Use filter-repo blob callback instead of filtering via git lfs ls-files (#4551)

When an LFS file is renamed (e.g. `libHarfBuzzSharp.so.0.60830.0` to `libHarfBuzzSharp.so`) the LFS blob stays the same. `git lfs ls-files --all --deleted` deduplicates by blob OID, so it only reports the new filename.
The old filename disappears from the list, changing historical commits and cascading a full history rewrite.

Replace the path-based LFS exclusion with inline blob content detection using git-filter-repo's `blob_callback`.
Any blob starting with the LFS pointer header is stripped regardless of  filename, making the filter immune to renames

Author: lolleko

engine/Tools/SboxBuild/Steps/SyncPublicRepo.cs

@@ -194,15 +194,16 @@ private bool SyncToPublicRepository()
 				return false;
 			}
 
-			// Make sure we filter out lfs files that are in the history as well
-			var allLfsPaths = GetAllPublicLfsFiles( relativeFilteredPath );
-			if ( allLfsPaths is null )
+			// Run git-filter-repo to filter out unwanted paths.
+			// LFS pointer blobs are detected and stripped inline by the Python
+			// filter (blob content inspection) so we no longer need to pass a
+			// pre-computed LFS path list.
+			if ( !RunFilterRepo( relativeFilteredPath ) )
 			{
 				return false;
 			}
 
-			// Run git-filter-repo to filter out unwanted paths
-			if ( !RunFilterRepo( relativeFilteredPath, allLfsPaths ) )
+			if ( !ValidateFilteredRepository( relativeFilteredPath ) )
 			{
 				return false;
 			}
@@ -391,7 +392,7 @@ private static bool TryUploadLfsArtifacts( string repoRoot, IReadOnlyCollection<
 		return TryUploadArtifacts( candidates, remoteBase, artifacts, uploadedHashes, "LFS", skipUpload );
 	}
 
-	private bool RunFilterRepo( string relativeRepoPath, IReadOnlyCollection<string> lfsPaths )
+	private bool RunFilterRepo( string relativeRepoPath )
 	{
 		Log.Info( "Running git-filter-repo to filter paths..." );
 
@@ -407,11 +408,7 @@ private bool RunFilterRepo( string relativeRepoPath, IReadOnlyCollection<string>
 			IncludeGlobs = RepoFilterPathIncludeGlobs,
 			ExcludeGlobs = RepoFilterPathExcludeGlobs,
 			WhitelistedShaders = RepoFilterShaderWhitelistGlobs,
-			PathRenames = RepoFilterPathRenames.ToDictionary( pair => pair.Key, pair => pair.Value, StringComparer.OrdinalIgnoreCase ),
-			LfsPaths = lfsPaths
-				.Select( ToForwardSlash )
-				.Distinct( StringComparer.OrdinalIgnoreCase )
-				.ToList()
+			PathRenames = RepoFilterPathRenames.ToDictionary( pair => pair.Key, pair => pair.Value, StringComparer.OrdinalIgnoreCase )
 		};
 
 		string configPath = null;
@@ -441,6 +438,54 @@ private bool RunFilterRepo( string relativeRepoPath, IReadOnlyCollection<string>
 		}
 	}
 
+	private static readonly HashSet<string> ForbiddenRepoExtensions = new( StringComparer.OrdinalIgnoreCase )
+	{
+		".lib", ".exe", ".pdb", ".a", ".dll", ".dylib", ".so",
+		".png", ".tga", ".jpg", ".psd", ".pdf", ".bmp", ".gif", ".exr", ".ico", ".svg", ".tif", ".tiff",
+		".ttf", ".otf",
+		".dmx", ".fbx", ".max",
+		".wav", ".ogg", ".mp3", ".mp4", ".webm", ".avi",
+		".pyd", ".ppf", ".vsix", ".vcs", ".bin", ".dat", ".jar", ".spv", ".ma", ".lxo"
+	};
+
+	private static bool ValidateFilteredRepository( string relativeRepoPath )
+	{
+		Log.Info( "Validating filtered repository before push..." );
+
+		var renamedTargets = new HashSet<string>( RepoFilterPathRenames.Values, StringComparer.OrdinalIgnoreCase );
+		var matcher = RepoFileFilter();
+		var violations = new List<string>();
+
+		Utility.RunProcess( "git", "ls-tree -r --name-only HEAD", relativeRepoPath, onDataReceived: ( _, e ) =>
+		{
+			if ( string.IsNullOrWhiteSpace( e.Data ) )
+				return;
+
+			var file = ToForwardSlash( e.Data.Trim() );
+
+			if ( file.StartsWith( "src/", StringComparison.OrdinalIgnoreCase ) )
+				violations.Add( $"Private source code: {file}" );
+
+			if ( !renamedTargets.Contains( file ) && !matcher.Match( file ).HasMatches )
+				violations.Add( $"Outside include rules: {file}" );
+
+			var ext = Path.GetExtension( file );
+			if ( !string.IsNullOrEmpty( ext ) && ForbiddenRepoExtensions.Contains( ext ) )
+				violations.Add( $"Forbidden extension ({ext}): {file}" );
+		} );
+
+		if ( violations.Count > 0 )
+		{
+			Log.Error( $"Filtered repository contains {violations.Count} violation(s):" );
+			foreach ( var v in violations )
+				Log.Error( $"  {v}" );
+			return false;
+		}
+
+		Log.Info( "Filtered repository validation passed" );
+		return true;
+	}
+
 	private string PushToPublicRepository( string relativeRepoPath )
 	{
 		Log.Info( "Pushing filtered repository to public..." );
@@ -531,26 +576,6 @@ private static HashSet<string> GetCurrentLfsFiles( string relativeRepoPath )
 		return trackedFiles;
 	}
 
-	private static HashSet<string> GetAllPublicLfsFiles( string relativeRepoPath )
-	{
-		var trackedFiles = GetCurrentLfsFiles( relativeRepoPath );
-
-		if ( !Utility.RunProcess( "git", "lfs ls-files --all --deleted --name-only", relativeRepoPath, onDataReceived: ( _, e ) =>
-		{
-			if ( string.IsNullOrWhiteSpace( e.Data ) )
-			{
-				return;
-			}
-
-			trackedFiles.Add( ToForwardSlash( e.Data.Trim() ) );
-		} ) )
-		{
-			Log.Error( "Failed to list historical LFS tracked files" );
-			return null;
-		}
-
-		return trackedFiles;
-	}
 
 	private void WriteDryRunOutputs( string commitHash, IEnumerable<ArtifactFileInfo> artifacts )
 	{
@@ -848,8 +873,5 @@ private sealed class FilterConfigData
 
 		[JsonPropertyName( "path_renames" )]
 		public Dictionary<string, string> PathRenames { get; init; }
-
-		[JsonPropertyName( "lfs_paths" )]
-		public List<string> LfsPaths { get; init; }
 	}
 }

engine/Tools/SboxBuild/Steps/SyncPublicRepoFilter.py

@@ -4,13 +4,17 @@
 
 import argparse
 import json
+import posixpath
 import sys
 from pathlib import PurePosixPath
 from typing import Dict, Iterable, List, Optional, Set
 import git_filter_repo as fr
 
+_LFS_POINTER_PREFIX = b"version https://git-lfs.github.com/spec/v1"
+
+
 class FilenameFilter:
-    """Applies include/exclude rules and renames for git-filter-repo."""
+    """Applies include/exclude rules and renames."""
 
     def __init__(self, config: Dict[str, object]) -> None:
         self._include_globs = tuple(_normalise_glob(p) for p in config.get("include_globs", []) or [])
@@ -23,17 +27,11 @@ def __init__(self, config: Dict[str, object]) -> None:
             for src, dest in renames.items()
         }
 
-        lfs_paths: Iterable[str] = config.get("lfs_paths", []) or []
-        self._lfs_paths: Set[str] = { _normalise_path(path) for path in lfs_paths }
-
     def __call__(self, filename: bytes) -> Optional[bytes]:
         path_text = filename.decode("utf-8", "ignore")
         normalised = _normalise_path(path_text)
         path = PurePosixPath(normalised)
 
-        if normalised in self._lfs_paths:
-            return None
-
         allowed = _matches_any_glob(path, self._include_globs)
 
         if allowed and _matches_any_glob(path, self._exclude_globs):
@@ -52,15 +50,65 @@ def __call__(self, filename: bytes) -> Optional[bytes]:
         return filename
 
 
+class LfsPointerFilter:
+    """Strips LFS pointer blobs and dangling symlinks from commits."""
+
+    def __init__(self) -> None:
+        self._lfs_blob_ids: Set[int] = set()
+        self._symlink_targets: Dict[int, str] = {}
+        self._stripped_paths: Set[str] = set()
+
+    def blob_callback(self, blob, _metadata) -> None:
+        if blob.data.startswith(_LFS_POINTER_PREFIX):
+            self._lfs_blob_ids.add(blob.id)
+        elif len(blob.data) < 512:
+            try:
+                self._symlink_targets[blob.id] = blob.data.decode("utf-8").rstrip("\n")
+            except UnicodeDecodeError:
+                pass
+
+    def strip_lfs_from_commit(self, commit) -> None:
+        original = commit.file_changes
+
+        stripped_this_commit: Set[str] = set()
+        after_lfs = []
+        for change in original:
+            if change.blob_id in self._lfs_blob_ids:
+                stripped_this_commit.add(change.filename.decode("utf-8", "replace"))
+                continue
+            after_lfs.append(change)
+
+        filtered = []
+        for change in after_lfs:
+            if change.mode == b"120000" and change.blob_id in self._symlink_targets:
+                target = self._symlink_targets[change.blob_id]
+                symlink_dir = PurePosixPath(change.filename.decode("utf-8", "replace")).parent
+                resolved = posixpath.normpath(str(symlink_dir / target))
+                if resolved in stripped_this_commit:
+                    stripped_this_commit.add(change.filename.decode("utf-8", "replace"))
+                    continue
+            filtered.append(change)
+
+        self._stripped_paths.update(stripped_this_commit)
+        commit.file_changes = filtered
+
+    def log_summary(self) -> None:
+        print(f"[LfsPointerFilter] Detected {len(self._lfs_blob_ids)} LFS pointer blob(s)")
+        print(f"[LfsPointerFilter] Stripped {len(self._stripped_paths)} unique path(s) from history")
+        if self._stripped_paths:
+            for path in sorted(self._stripped_paths):
+                print(f"  - {path}")
+
+
 class BaselineCommitCallback:
-    """Rewrites the root commit metadata for the public history."""
+    """Rewrites the root commit metadata."""
 
     _base_message = (
         "Open source release\n\n"
         "This commit imports the C# engine code and game files, excluding C++ source code."
     )
 
-    def __call__(self, commit, metadata) -> None:  # pylint: disable=unused-argument
+    def __call__(self, commit, metadata) -> None:
         if commit.parents:
             return
 
@@ -88,26 +136,33 @@ def _matches_any_glob(path: PurePosixPath, patterns: Iterable[str]) -> bool:
 
 
 def main(argv: Optional[List[str]] = None) -> int:
-    parser = argparse.ArgumentParser(description="Run git-filter-repo with s&box filters")
-    parser.add_argument("--config", required=True, help="Path to JSON configuration file")
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--config", required=True)
     args = parser.parse_args(argv)
 
     with open(args.config, "r", encoding="utf-8") as fp:
         config = json.load(fp)
 
     filename_filter = FilenameFilter(config)
-    commit_callback = BaselineCommitCallback()
+    baseline_callback = BaselineCommitCallback()
+    lfs_filter = LfsPointerFilter()
+
+    def commit_callback(commit, metadata):
+        lfs_filter.strip_lfs_from_commit(commit)
+        baseline_callback(commit, metadata)
 
     options = fr.FilteringOptions.parse_args([], error_on_empty=False)
     options.force = True
 
     repo_filter = fr.RepoFilter(
         options,
+        blob_callback=lfs_filter.blob_callback,
         filename_callback=filename_filter,
         commit_callback=commit_callback,
     )
 
     repo_filter.run()
+    lfs_filter.log_summary()
     return 0
 
 if __name__ == "__main__":