Automated Test improvements

TestCommon/TestFixtures.cs

@@ -4,6 +4,9 @@
 
 namespace UnityDataTools.TestCommon;
 
+// Base class that facilitates iterating through sub-sub-folders
+// inside the Data location.  E.g. GetContexts("AssetBundles")
+// finds "TestCommon/Data/AssetBundles/2019.4.0f1", "TestCommon/Data/AssetBundles/2020.3.0f1" etc.
 public class BaseTestFixture
 {
     protected Context Context { get; }
@@ -15,6 +18,8 @@ public BaseTestFixture(Context context)
         Context = context;
     }
 
+    // Tests that have files that record the expected results for each version
+    // of Unity can override this method to regenerate those expected results.
     protected virtual void OnLoadExpectedData(Context context)
     {
     }
@@ -23,6 +28,9 @@ protected virtual void OnLoadExpectedData(Context context)
     public void LoadExpectedData()
     {
         OnLoadExpectedData(Context);
+
+        // Load json file with the expected results for a test based on
+        // folder structure convention (e.g. ExpectedData/<UnityVersion>/ExpectedVersions.json)
         Context.ExpectedData.Load(Context.ExpectedDataFolder);
     }
 
@@ -47,6 +55,9 @@ protected static IEnumerable<Context> GetContexts(string dataFolder)
     }
 }
 
+// Test fixture that repeats the tests for each folder inside TestCommon/Data/AssetBundles.
+// Each sub-folder is expected to have results of an AssetBundle build repeated with a
+// different version of Unity.
 [TestFixtureSource(typeof(AssetBundleTestFixture), nameof(GetContexts))]
 public class AssetBundleTestFixture : BaseTestFixture
 {

UnityDataTool.Tests/AddressablesBuildLayoutTests.cs

@@ -0,0 +1,68 @@
+using System;
+using Microsoft.Data.Sqlite;
+using System.IO;
+using System.Linq;
+using System.Threading.Tasks;
+using NUnit.Framework;
+
+namespace UnityDataTools.UnityDataTool.Tests;
+
+#pragma warning disable NUnit2005, NUnit2006
+
+public class AddressablesBuildLayoutTests
+{
+    private string m_TestOutputFolder;
+    private string m_TestDataFolder;
+
+    [OneTimeSetUp]
+    public void OneTimeSetup()
+    {
+        m_TestOutputFolder = Path.Combine(TestContext.CurrentContext.TestDirectory, "test_folder");
+        m_TestDataFolder = Path.Combine(TestContext.CurrentContext.TestDirectory, "Data");
+        Directory.CreateDirectory(m_TestOutputFolder);
+        Directory.SetCurrentDirectory(m_TestOutputFolder);
+    }
+
+    [TearDown]
+    public void Teardown()
+    {
+        SqliteConnection.ClearAllPools();
+
+        var testDir = new DirectoryInfo(m_TestOutputFolder);
+        testDir.EnumerateFiles()
+            .ToList().ForEach(f => f.Delete());
+        testDir.EnumerateDirectories()
+            .ToList().ForEach(d => d.Delete(true));
+    }
+
+    [Test]
+    public async Task Analyze_BuildLayout_ContainsExpectedSQLContent()
+    {
+        // This folder contains reference files from two builds of the "AudioExample"
+        // Addressables test project.
+        // The test confirms some expected content in the database
+        var path = Path.Combine(m_TestDataFolder, "AddressableBuildLayouts");
+        var databasePath = SQLTestHelper.GetDatabasePath(m_TestOutputFolder);
+
+        Assert.AreEqual(0, await Program.Main(new string[] { "analyze", path, "-p", "*.json" }));
+        using var db = SQLTestHelper.OpenDatabase(databasePath);
+
+        // Sanity check some expected content in the output SQLite database
+        SQLTestHelper.AssertQueryInt(db, "SELECT COUNT(*) FROM addressables_builds", 2,
+            "Unexpected number of builds");
+        SQLTestHelper.AssertQueryInt(db, "SELECT COUNT(*) FROM addressables_builds WHERE name = \"buildlayout_2025.01.28.16.35.01.json\"", 1,
+            "Failed to find build matching reference filename");
+        SQLTestHelper.AssertQueryString(db, "SELECT unity_version FROM addressables_builds WHERE id = 1", "6000.1.0b2",
+            "Unexpected Unity Version");
+        SQLTestHelper.AssertQueryString(db, "SELECT package_version FROM addressables_builds WHERE id = 1", "com.unity.addressables: 2.2.2",
+            "Unexpected Addressables version");
+        SQLTestHelper.AssertQueryInt(db, "SELECT COUNT(*) FROM addressables_build_bundles WHERE build_id = 1 and name = \"samplepack1_assets_0.bundle\"", 1,
+            "Expected to find specific AssetBundle by name");
+        SQLTestHelper.AssertQueryInt(db, "SELECT file_size FROM addressables_build_bundles WHERE build_id = 2 and name = \"samplepack1_assets_0.bundle\"", 33824,
+            "Unexpected size for specific AssetBundle in build 2");
+        SQLTestHelper.AssertQueryString(db, "SELECT packing_mode FROM addressables_build_groups WHERE build_id = 1 and name = \"SamplePack1\"", "PackSeparately",
+            "Unexpected packing_mode for group");
+        SQLTestHelper.AssertQueryInt(db, "SELECT COUNT(*) FROM asset_bundles", 0,
+            "Expected no AssetBundles found in reference folder");
+    }
+}

UnityDataTool.Tests/BuildReportTests.cs

@@ -0,0 +1,215 @@
+using Microsoft.Data.Sqlite;
+using System.IO;
+using System.Linq;
+using System.Threading.Tasks;
+using NUnit.Framework;
+using System.Collections.Generic;
+
+namespace UnityDataTools.UnityDataTool.Tests;
+
+#pragma warning disable NUnit2005, NUnit2006
+
+public class BuildReportTests
+{
+    private string m_TestOutputFolder;
+    private string m_TestDataFolder;
+
+    [OneTimeSetUp]
+    public void OneTimeSetup()
+    {
+        m_TestOutputFolder = Path.Combine(TestContext.CurrentContext.TestDirectory, "test_folder");
+        m_TestDataFolder = Path.Combine(TestContext.CurrentContext.TestDirectory, "Data");
+        Directory.CreateDirectory(m_TestOutputFolder);
+        Directory.SetCurrentDirectory(m_TestOutputFolder);
+    }
+
+    [TearDown]
+    public void Teardown()
+    {
+        SqliteConnection.ClearAllPools();
+
+        var testDir = new DirectoryInfo(m_TestOutputFolder);
+        testDir.EnumerateFiles()
+            .ToList().ForEach(f => f.Delete());
+        testDir.EnumerateDirectories()
+            .ToList().ForEach(d => d.Delete(true));
+    }
+
+    // Check the primary object/file tables and views which are populated by the general
+    // object handling of the analyzer (e.g. nothing BuildReport specific)
+    // This test is parameterized to run with and without "--skip-references"
+    // in order to show that the core object tables are not impacted by whether
+    // or not references are tracked.
+    [Test]
+    public async Task Analyze_BuildReport_ContainsExpected_ObjectInfo(
+        [Values(false, true)] bool skipReferences)
+    {
+        // This folder contains a reference build report generated by a build of the TestProject
+        // in the BuildReportInspector package.
+        var path = Path.Combine(m_TestDataFolder, "BuildReport1");
+        var databasePath = SQLTestHelper.GetDatabasePath(m_TestOutputFolder);
+
+        var args = new List<string> { "analyze", path, "-p", "*.buildreport" };
+        if (skipReferences)
+            args.Add("--skip-references");
+
+        Assert.AreEqual(0, await Program.Main(args.ToArray()));
+        using var db = SQLTestHelper.OpenDatabase(databasePath);
+
+        // Sanity check the Unity objects found in this Build report file
+        // Tip: The meaning of the hard coded type ids used in the queries can be found
+        // at https://docs.unity3d.com/6000.3/Documentation/Manual/ClassIDReference.html
+
+        // The BuildReport object is the most important.
+        // PackedAssets objects are present for each output serialized file, .resS and .resource.
+        const int packedAssetCount = 7;
+
+        SQLTestHelper.AssertQueryInt(db, "SELECT COUNT(*) FROM objects WHERE type = 1125", 1,
+            "Unexpected number of BuildReport objects (type 1125)");
+        SQLTestHelper.AssertQueryInt(db, "SELECT COUNT(*) FROM objects WHERE type = 1126", packedAssetCount,
+            "Unexpected number of PackedAssets objects");
+
+        // This object is expected inside AssetBundle builds
+        SQLTestHelper.AssertQueryInt(db, "SELECT COUNT(*) FROM objects WHERE type = 668709126", 1,
+            "Unexpected number of BuiltAssetBundleInfoSet objects");
+
+        // There can be other more obscure objects present, depending on the build,
+        // e.g. PluginBuildInfo, AudioBuildInfo, VideoBuildInfo etc.
+        var ttlObjCount = SQLTestHelper.QueryInt(db, "SELECT COUNT(*) FROM objects");
+        Assert.That(ttlObjCount, Is.GreaterThanOrEqualTo(1+ packedAssetCount + 1),
+            "Unexpected number of objects in BuildReport analysis");
+
+        SQLTestHelper.AssertQueryInt(db, "SELECT COUNT(*) FROM asset_bundles", 0,
+            "Expected no AssetBundles found in reference folder");
+
+        //
+        // Tests using object_view which lets us refer to objects by type name
+        //
+        SQLTestHelper.AssertQueryInt(db, "SELECT COUNT(*) FROM object_view WHERE type = 'BuildReport'", 1,
+            "Expected exactly one BuildReport in object_view");
+
+        SQLTestHelper.AssertQueryString(db, "SELECT name FROM object_view WHERE type = 'BuildReport'", "Build AssetBundles",
+            "Unexpected name");
+
+        SQLTestHelper.AssertQueryString(db, "SELECT name FROM object_view WHERE type = 'BuildReport'", "Build AssetBundles",
+            "Unexpected BuildReport name in object_view");
+
+        SQLTestHelper.AssertQueryInt(db, "SELECT COUNT(*) FROM object_view WHERE type = 'PackedAssets'", packedAssetCount,
+            "Unexpected number of PackedAssets in object_view");
+
+        SQLTestHelper.AssertQueryInt(db, "SELECT COUNT(*) FROM object_view WHERE type = 'BuiltAssetBundleInfoSet'", 1,
+            "Expected exactly one BuiltAssetBundleInfoSet in object_view");
+
+        // Verify all rows have the same serialized_file
+        SQLTestHelper.AssertQueryInt(db, "SELECT COUNT(DISTINCT serialized_file) FROM object_view", 1,
+            "All objects should be from the same serialized file");
+
+        SQLTestHelper.AssertQueryString(db, "SELECT DISTINCT serialized_file FROM object_view", "LastBuild.buildreport",
+            "Unexpected serialized file name in object_view");
+
+        // Verify the BuildReport object has expected properties
+        var buildReportSize = SQLTestHelper.QueryInt(db, "SELECT size FROM object_view WHERE type = 'BuildReport'");
+        Assert.That(buildReportSize, Is.GreaterThan(0), "BuildReport size should be greater than 0");
+
+        //
+        // Tests using view_breakdown_by_type which aggregates objects by type
+        //
+
+        // Verify counts match for specific types
+        SQLTestHelper.AssertQueryInt(db, "SELECT count FROM view_breakdown_by_type WHERE type = 'BuildReport'", 1,
+            "Expected 1 BuildReport in breakdown view");
+        SQLTestHelper.AssertQueryInt(db, "SELECT count FROM view_breakdown_by_type WHERE type = 'PackedAssets'", packedAssetCount,
+            "Expected 7 PackedAssets in breakdown view");
+
+        var buildReportSize2 = SQLTestHelper.QueryInt(db, "SELECT byte_size FROM view_breakdown_by_type WHERE type = 'BuildReport'");
+        Assert.AreEqual(buildReportSize, buildReportSize2, "Mismatch between object_view and breakdown_view for BuildReport size");
+
+        // Verify pretty_size formatting exists
+        var buildReportPrettySize = SQLTestHelper.QueryString(db, "SELECT pretty_size FROM view_breakdown_by_type WHERE type = 'BuildReport'");
+        Assert.That(buildReportPrettySize, Does.Contain("KB").Or.Contain("B"), "BuildReport pretty_size should have size unit");
+
+        // Verify total byte_size across all types
+        var totalSize = SQLTestHelper.QueryInt(db, "SELECT SUM(byte_size) FROM view_breakdown_by_type");
+        Assert.That(totalSize, Is.GreaterThan(buildReportSize),
+            "Unexpected number of objects in BuildReport analysis");
+
+        //
+        // Tests using serialized_files table
+        //
+        SQLTestHelper.AssertQueryInt(db, "SELECT COUNT(*) FROM serialized_files", 1,
+            "Expected exactly one serialized file");
+
+        SQLTestHelper.AssertQueryString(db, "SELECT name FROM serialized_files WHERE id = 0", "LastBuild.buildreport",
+            "Unexpected serialized file name");
+
+        // Verify asset_bundle column is empty/NULL for BuildReport files (they are not asset bundles)
+        var assetBundleValue = SQLTestHelper.QueryString(db, "SELECT COALESCE(asset_bundle, '') FROM serialized_files WHERE id = 0");
+        Assert.That(string.IsNullOrEmpty(assetBundleValue), "BuildReport serialized file should not have asset_bundle value");
+
+        // Verify the serialized file name matches what we see in object_view
+        var serializedFileName = SQLTestHelper.QueryString(db, "SELECT name FROM serialized_files WHERE id = 0");
+        var objectViewFileName = SQLTestHelper.QueryString(db, "SELECT DISTINCT serialized_file FROM object_view");
+        Assert.AreEqual(serializedFileName, objectViewFileName,
+            "Serialized file name should match between serialized_files table and object_view");
+    }
+
+    // The BuildReport file has a simple structure with a single BuildReport object
+    // and all other objects referenced from its Appendicies array.
+    // This gives an opportunity for a detailed test that the "refs" table is properly populated.
+    [Test]
+    public async Task Analyze_BuildReport_ContainsExpectedReferences(
+        [Values(false, true)] bool skipReferences)
+    {
+        var path = Path.Combine(m_TestDataFolder, "BuildReport1");
+        var databasePath = SQLTestHelper.GetDatabasePath(m_TestOutputFolder);
+
+        var args = new List<string> { "analyze", path, "-p", "*.buildreport" };
+        if (skipReferences)
+            args.Add("--skip-references");
+
+        Assert.AreEqual(0, await Program.Main(args.ToArray()));
+        using var db = SQLTestHelper.OpenDatabase(databasePath);
+
+        if (skipReferences)
+        {
+            // When --skip-references is used, the refs table should be empty
+            SQLTestHelper.AssertQueryInt(db, "SELECT COUNT(*) FROM refs", 0,
+                "refs table should be empty when --skip-references is used");
+            return;
+        }
+
+        var buildReportId = SQLTestHelper.QueryInt(db,
+            "SELECT id FROM objects WHERE type = 1125");
+
+        var totalObjectCount = SQLTestHelper.QueryInt(db, "SELECT COUNT(*) FROM objects");
+
+        var expectedRefCount = totalObjectCount - 1;
+        SQLTestHelper.AssertQueryInt(db, "SELECT COUNT(*) FROM refs", expectedRefCount,
+            "BuildReport should reference all other objects");
+
+        SQLTestHelper.AssertQueryInt(db, $"SELECT COUNT(*) FROM refs WHERE object = {buildReportId}", expectedRefCount,
+            "All references should originate from BuildReport object");
+
+        SQLTestHelper.AssertQueryInt(db, $"SELECT COUNT(*) FROM refs WHERE referenced_object = {buildReportId}", 0,
+            "No object should reference the BuildReport object");
+
+        var refsWithWrongPath = SQLTestHelper.QueryInt(db,
+            "SELECT COUNT(*) FROM refs WHERE property_path NOT LIKE 'm_Appendices[%]'");
+        Assert.AreEqual(0, refsWithWrongPath, "All property_path values should match pattern 'm_Appendices[N]'");
+
+        SQLTestHelper.AssertQueryString(db, "SELECT DISTINCT property_type FROM refs", "Object",
+            "All references should have property_type 'Object'");
+
+        var objectsNotReferenced = SQLTestHelper.QueryInt(db,
+            $@"SELECT COUNT(*) FROM objects
+                WHERE id != {buildReportId}
+                AND id NOT IN (SELECT referenced_object FROM refs)");
+        Assert.AreEqual(0, objectsNotReferenced,
+            "Every object except BuildReport should be referenced exactly once");
+
+        var duplicateRefs = SQLTestHelper.QueryInt(db,
+            "SELECT COUNT(*) FROM (SELECT referenced_object, COUNT(*) as cnt FROM refs GROUP BY referenced_object HAVING cnt > 1)");
+        Assert.AreEqual(0, duplicateRefs,
+            "No object should be referenced more than once");
+    }
+}

UnityDataTool.Tests/ExpectedDataGenerator.cs

@@ -5,6 +5,10 @@
 
 namespace UnityDataTools.UnityDataTool.Tests;
 
+// Collect and record the current output returned by the same UnityDataTool commands
+// that the tests will run.  Once saved these become the reference data, and if the output
+// changes the tests will fail.  So this can be repeated if there is an "expected" change
+// in the output.
 public static class ExpectedDataGenerator
 {
     public static void Generate(Context context)
@@ -71,6 +75,9 @@ public static void Generate(Context context)
         var csprojFolder = Directory.GetParent(context.TestDataFolder).Parent.Parent.Parent.FullName;
         var outputFolder = Path.Combine(csprojFolder, "ExpectedData", context.UnityDataVersion);
 
+        expectedData.Save(outputFolder);
+
+        // Also take a snapshot of the output of running "dump" commands on the test file "assetbundle"
         Directory.CreateDirectory(outputFolder);
 
         var dumpPath = Path.Combine(outputFolder, "dump");
@@ -80,7 +87,5 @@ public static void Generate(Context context)
         dumpPath = Path.Combine(outputFolder, "dump-s");
         Directory.CreateDirectory(dumpPath);
         Program.Main(new string[] { "dump", Path.Combine(context.UnityDataFolder, "assetbundle"), "-o", dumpPath, "-s" });
-
-        expectedData.Save(outputFolder);
     }
 }