Bone tracks, IK controls, character anim tools (#3021)

Documentation: https://sbox.game/dev/doc/systems/movie-maker/skeletal-animation/

Author: Metapyziks

engine/Sandbox.Engine/Systems/Movies/Binder/Binder.cs

@@ -82,6 +82,36 @@ private ITrackTarget CreateTarget( ITrack track, ITrackTarget? parent = null )
 		};
 	}
 
+	/// <summary>
+	/// Get all reference targets for tracks in the given <paramref name="clip"/>.
+	/// </summary>
+	/// <typeparam name="T">Reference type, for example <see cref="GameObject"/> or a <see cref="Component"/> type.</typeparam>
+	/// <param name="clip">Movie clip to find track bindings for.</param>
+	public IEnumerable<ITrackReference<T>> GetReferences<T>( IMovieClip clip )
+		where T : Component
+	{
+		return clip.Tracks
+			.OfType<IReferenceTrack<T>>()
+			.Select( Get );
+	}
+
+	/// <summary>
+	/// Get all property targets for tracks in the given <paramref name="clip"/>.
+	/// </summary>
+	/// <typeparam name="T">Property value type.</typeparam>
+	/// <param name="clip">Movie clip to find track bindings for.</param>
+	public IEnumerable<ITrackProperty<T>> GetProperties<T>( IMovieClip clip )
+	{
+		return clip.Tracks
+			.OfType<IPropertyTrack<T>>()
+			.Select( Get );
+	}
+
+	/// <summary>
+	/// Get all bound component references for tracks in the given <paramref name="clip"/>.
+	/// </summary>
+	/// <typeparam name="T">Component type.</typeparam>
+	/// <param name="clip">Movie clip to find track bindings for.</param>
 	public IEnumerable<T> GetComponents<T>( IMovieClip clip )
 		where T : Component
 	{

engine/Sandbox.Engine/Systems/Movies/Binder/Properties/AnimParam.cs

@@ -33,8 +33,6 @@ public T Value
 [Expose]
 file sealed class AnimParamPropertyFactory : ITrackPropertyFactory<ITrackProperty<ParameterAccessor?>>
 {
-	string ITrackPropertyFactory.CategoryName => "Anim Graph";
-
 	public IEnumerable<string> GetPropertyNames( ITrackProperty<ParameterAccessor?> parent )
 	{
 		var graph = parent is { IsBound: true } ? parent.Value?.Graph : null;
@@ -47,6 +45,8 @@ public IEnumerable<string> GetPropertyNames( ITrackProperty<ParameterAccessor?>
 		}
 	}
 
+	public string GetCategoryName( ITrackProperty<ParameterAccessor?> parent, string name ) => "Anim Graph";
+
 	/// <summary>
 	/// Any property in a <see cref="ParameterAccessor"/> is an anim graph parameter, but we
 	/// can only determine the type if it actually exists.

engine/Sandbox.Engine/Systems/Movies/Binder/Properties/Bone.cs

@@ -0,0 +1,274 @@
+using System.Runtime.CompilerServices;
+
+namespace Sandbox.MovieMaker.Properties;
+
+#nullable enable
+
+/// <summary>
+/// Pseudo-property on a <see cref="SkinnedModelRenderer"/> that has a sub-property for each bone.
+/// Stores movie-driven transforms for each bone during playback, and applies them when
+/// <see cref="MovieBoneAnimatorSystem"/> performs <see cref="GameObjectSystem.Stage.UpdateBones"/>.
+/// </summary>
+[Expose]
+public sealed class BoneAccessor
+{
+	private readonly Dictionary<int, Transform> _parentSpaceOverrides = new();
+	private readonly Dictionary<int, Transform> _localSpaceOverrides = new();
+	private readonly SkinnedModelRenderer _renderer;
+
+	/// <summary>
+	/// Renderer this accessor was created for.
+	/// </summary>
+	public SkinnedModelRenderer Renderer => _renderer;
+
+	public BoneAccessor( SkinnedModelRenderer renderer )
+	{
+		_renderer = renderer;
+	}
+
+	/// <summary>
+	/// Helper to see if the renderer's model has a bone with the given <paramref name="name"/>.
+	/// </summary>
+	public bool HasBone( string name ) => _renderer.Model?.Bones.HasBone( name ) ?? false;
+
+	/// <summary>
+	/// Gets the current movie-driven parent-space transform of the given bone. If the bone
+	/// isn't controlled by a movie, just returns the current parent-space transform.
+	/// </summary>
+	public Transform GetParentSpace( int index )
+	{
+		return _parentSpaceOverrides.TryGetValue( index, out var transform )
+			? transform
+			: Renderer.SceneModel?.GetParentSpaceBone( index ) ?? Transform.Zero;
+	}
+
+	/// <summary>
+	/// Sets the current movie-driven parent-space transform of the given bone.
+	/// </summary>
+	public void SetParentSpace( int index, Transform value )
+	{
+		_parentSpaceOverrides[index] = value;
+	}
+
+	/// <summary>
+	/// Clears any movie-driven bone transforms for this renderer.
+	/// </summary>
+	public void ClearOverrides()
+	{
+		_parentSpaceOverrides.Clear();
+	}
+
+	/// <summary>
+	/// Applies any movie-driven bone transforms. Called during <see cref="GameObjectSystem.Stage.UpdateBones"/>.
+	/// </summary>
+	public void ApplyOverrides()
+	{
+		_renderer.ClearPhysicsBones();
+
+		if ( _renderer.Model is not { } model ) return;
+		if ( _renderer.SceneModel is not { } sceneModel ) return;
+		if ( _parentSpaceOverrides.Count == 0 ) return;
+
+		// TODO: I'm assuming parent bones are always listed before child bones
+
+		_localSpaceOverrides.Clear();
+
+		foreach ( var bone in model.Bones.AllBones )
+		{
+			if ( !_parentSpaceOverrides.TryGetValue( bone.Index, out var parentLocalTransform ) )
+			{
+				// Even if this bone doesn't have an override, one of its ancestors
+				// might have so we need to update its local space transform
+
+				parentLocalTransform = bone.Parent is { } parent
+					? parent.LocalTransform.ToLocal( bone.LocalTransform )
+					: bone.LocalTransform;
+			}
+
+			{
+				var parentTransform = bone.Parent is { } parent
+					? _localSpaceOverrides.TryGetValue( parent.Index, out var parentLocalOverride )
+						? parentLocalOverride
+						: sceneModel.GetBoneLocalTransform( parent.Index )
+					: Transform.Zero;
+
+				var localTransform = parentTransform.ToWorld( parentLocalTransform );
+
+				_localSpaceOverrides[bone.Index] = localTransform;
+
+				sceneModel.SetBoneOverride( bone.Index, localTransform );
+			}
+		}
+	}
+}
+
+/// <summary>
+/// Reads / writes a bone transform on a <see cref="SkinnedModelRenderer"/>.
+/// </summary>
+file sealed record BoneProperty( ITrackProperty<BoneAccessor?> Parent, string Name )
+	: ITrackProperty<Transform>
+{
+	private (SkinnedModelRenderer? Renderer, int? Index)? _cached;
+
+	public bool IsBound => Parent.Value?.HasBone( Name ) ?? false;
+
+	public Transform Value
+	{
+		get => GetInfo().Index is not { } index ? Transform.Zero : Parent.Value?.GetParentSpace( index ) ?? Transform.Zero;
+		set
+		{
+			if ( GetInfo().Index is { } index )
+			{
+				Parent.Value?.SetParentSpace( index, value );
+			}
+		}
+	}
+
+	ITrackTarget ITrackProperty.Parent => Parent;
+
+	private (SkinnedModelRenderer? Renderer, int? Index) GetInfo()
+	{
+		if ( _cached is { } cached && cached.Renderer == Parent.Value?.Renderer )
+		{
+			return cached;
+		}
+
+		var renderer = Parent.Value?.Renderer;
+		var index = renderer?.Model?.Bones.GetBone( Name )?.Index;
+
+		_cached = cached = (renderer, index);
+
+		return cached;
+	}
+}
+
+[Expose]
+file sealed class BonePropertyFactory : ITrackPropertyFactory<ITrackProperty<BoneAccessor?>, Transform>
+{
+	/// <summary>
+	/// Any property inside a <see cref="BoneAccessor"/> is a bone.
+	/// </summary>
+	public bool PropertyExists( ITrackProperty<BoneAccessor?> parent, string name ) => true;
+
+	public ITrackProperty<Transform> CreateProperty( ITrackProperty<BoneAccessor?> parent, string name ) => new BoneProperty( parent, name );
+
+	public IEnumerable<string> GetPropertyNames( ITrackProperty<BoneAccessor?> parent )
+	{
+		return parent is { IsBound: true, Value.Renderer.Model: { } model }
+			? model.Bones.AllBones.Select( x => x.Name )
+			: [];
+	}
+
+	public string GetCategoryName( ITrackProperty<BoneAccessor?> parent, string name )
+	{
+		var firstSep = name.IndexOf( '_' );
+		var lastSep = name.LastIndexOf( '_' );
+
+		if ( firstSep == lastSep ) return "Bones";
+
+		return $"Bones/{name[..firstSep].ToTitleCase()}";
+	}
+}
+
+file sealed record BoneAccessorProperty( ITrackReference<SkinnedModelRenderer> Parent )
+	: ITrackProperty<BoneAccessor?>
+{
+	public const string PropertyName = "Bones";
+
+	public string Name => PropertyName;
+
+	public BoneAccessor? Value
+	{
+		get => Parent.Value is { } renderer
+			? MovieBoneAnimatorSystem.Current?.GetBoneAccessor( renderer )
+			: null;
+
+		set
+		{
+			// Can't write (CanWrite = false)
+		}
+	}
+
+	bool ITrackProperty.CanWrite => false;
+
+	ITrackTarget ITrackProperty.Parent => Parent;
+}
+
+[Expose]
+file sealed class BoneAccessorPropertyFactory : ITrackPropertyFactory<ITrackReference<SkinnedModelRenderer>, BoneAccessor?>
+{
+	public string GetCategoryName( ITrackReference<SkinnedModelRenderer> parent, string name ) => "Members";
+
+	public IEnumerable<string> GetPropertyNames( ITrackReference<SkinnedModelRenderer> parent ) =>
+		[BoneAccessorProperty.PropertyName];
+
+	public bool PropertyExists( ITrackReference<SkinnedModelRenderer> parent, string name ) =>
+		name == BoneAccessorProperty.PropertyName;
+
+	public ITrackProperty<BoneAccessor?> CreateProperty( ITrackReference<SkinnedModelRenderer> parent, string name ) =>
+		new BoneAccessorProperty( parent );
+}
+
+/// <summary>
+/// Coordinates playing bone animations from <see cref="MoviePlayer"/>s. Holds a <see cref="BoneAccessor"/>
+/// for <see cref="SkinnedModelRenderer"/>s in the scene, which store any movie-controlled bone transforms.
+/// </summary>
+[Expose]
+public sealed class MovieBoneAnimatorSystem : GameObjectSystem<MovieBoneAnimatorSystem>
+{
+	private readonly ConditionalWeakTable<SkinnedModelRenderer, BoneAccessor> _accessors = new();
+
+	public MovieBoneAnimatorSystem( Scene scene ) : base( scene )
+	{
+		Listen( Stage.UpdateBones, -1_000, UpdateBones, "UpdateBones" );
+	}
+
+	/// <summary>
+	/// Applies any active movie-driven bone transformations.
+	/// </summary>
+	public void UpdateBones()
+	{
+		foreach ( var (_, accessor) in _accessors )
+		{
+			accessor.ApplyOverrides();
+		}
+	}
+
+	/// <summary>
+	/// Clears all movie-driven bone transformations for the given <paramref name="renderer"/>.
+	/// </summary>
+	public void ClearBones( SkinnedModelRenderer renderer )
+	{
+		if ( _accessors.TryGetValue( renderer, out var accessor ) )
+		{
+			accessor.ClearOverrides();
+		}
+	}
+
+	/// <summary>
+	/// Gets the current movie-driven parent-space transform for the given bone. If this
+	/// bone isn't currently being controlled by a movie, returns its current transform.
+	/// </summary>
+	public Transform GetParentSpaceBone( SkinnedModelRenderer renderer, int index )
+	{
+		return GetBoneAccessor( renderer ).GetParentSpace( index );
+	}
+
+	/// <summary>
+	/// Sets the current movie-driven parent-space transform for the given bone.
+	/// </summary>
+	public void SetParentSpaceBone( SkinnedModelRenderer renderer, int index, Transform transform )
+	{
+		GetBoneAccessor( renderer ).SetParentSpace( index, transform );
+	}
+
+	internal BoneAccessor GetBoneAccessor( SkinnedModelRenderer renderer )
+	{
+		if ( _accessors.TryGetValue( renderer, out var existing ) ) return existing;
+
+		existing = new BoneAccessor( renderer );
+		_accessors.Add( renderer, existing );
+
+		return existing;
+	}
+}

engine/Sandbox.Engine/Systems/Movies/Binder/Properties/Member.cs

@@ -95,8 +95,6 @@ private bool IsBoneTransformProperty( [NotNullWhen( true )] out GameObject? bone
 [Expose]
 file sealed class MemberPropertyFactory : ITrackPropertyFactory
 {
-	string ITrackPropertyFactory.CategoryName => "Members";
-
 	int ITrackPropertyFactory.Order => 0x4000_0000;
 
 	IEnumerable<string> ITrackPropertyFactory.GetPropertyNames( ITrackTarget parent )
@@ -119,6 +117,8 @@ IEnumerable<string> ITrackPropertyFactory.GetPropertyNames( ITrackTarget parent
 			.FirstOrDefault( CanMakeTrackFromMember );
 	}
 
+	public string GetCategoryName( ITrackTarget parent, string name ) => "Members";
+
 	public Type? GetTargetType( ITrackTarget parent, string name )
 	{
 		return GetMember( parent, name ) switch

engine/Sandbox.Engine/Systems/Movies/Binder/Properties/Morph.cs

@@ -23,8 +23,6 @@ public float Value
 [Expose]
 file sealed class MorphPropertyFactory : ITrackPropertyFactory<ITrackProperty<MorphAccessor?>, float>
 {
-	string ITrackPropertyFactory.CategoryName => "Morphs";
-
 	/// <summary>
 	/// Any property inside a <see cref="MorphAccessor"/> is a morph.
 	/// </summary>
@@ -37,4 +35,6 @@ public IEnumerable<string> GetPropertyNames( ITrackProperty<MorphAccessor?> pare
 			? accessor.Names
 			: Enumerable.Empty<string>();
 	}
+
+	public string GetCategoryName( ITrackProperty<MorphAccessor?> parent, string name ) => "Morphs";
 }