[size-opt] prefix tree double check; zip level 9

Author: UlyssesWu

FreeMote/Consts.cs

@@ -61,7 +61,7 @@ public static class Consts
 
         /// <summary>
         /// (bool)
-        /// <para>Fast: 0x9C BestCompression: 0xDA NoCompression/Low: 0x01</para>
+        /// <para>Fast: 0x9C; BestCompression: 0xDA; NoCompression/Low: 0x01</para>
         /// </summary>
         public const string Context_PsbZlibFastCompress = "PsbZlibFastCompress";
 

FreeMote/FreeMote.csproj

@@ -27,6 +27,7 @@
     <PackageReference Include="Microsoft.IO.RecyclableMemoryStream">
       <Version>2.3.2</Version>
     </PackageReference>
+    <PackageReference Include="SharpZipLib" Version="1.4.2" />
     <PackageReference Include="System.Memory">
       <Version>4.5.5</Version>
     </PackageReference>

FreeMote/PrefixTree.cs

@@ -1,4 +1,4 @@
-using System;
+using System;
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.Linq;
@@ -124,11 +124,54 @@ internal void InsertTree(string value)
         public static PrefixTree Build(List<string> namesList, bool optimize, out List<uint> names, out List<uint> tree, out List<uint> offsets)
         {
             //namesList.Sort((s1, s2) => s1.Length - s2.Length);
-            var trie = new PrefixTree(namesList, optimize);
-            names = trie._names;
-            tree = trie._tree;
-            offsets = trie._offsets;
-            return trie;
+            if (!optimize)
+            {
+                var trie = new PrefixTree(namesList, false);
+                names = trie._names;
+                tree = trie._tree;
+                offsets = trie._offsets;
+                return trie;
+            }
+
+            // Some datasets are much denser with the legacy allocator.
+            // Keep OptimizeMode stable by choosing whichever layout serializes smaller.
+            var optimizedTrie = new PrefixTree(namesList, true);
+            var legacyTrie = new PrefixTree(namesList, false);
+            var optimizedSize = EstimateSerializedSize(optimizedTrie._names, optimizedTrie._tree, optimizedTrie._offsets);
+            var legacySize = EstimateSerializedSize(legacyTrie._names, legacyTrie._tree, legacyTrie._offsets);
+
+            var result = optimizedSize <= legacySize ? optimizedTrie : legacyTrie;
+            names = result._names;
+            tree = result._tree;
+            offsets = result._offsets;
+            return result;
+        }
+
+        private static int EstimateSerializedSize(List<uint> names, List<uint> tree, List<uint> offsets)
+            => EstimateArraySize(offsets) + EstimateArraySize(tree) + EstimateArraySize(names);
+
+        private static int EstimateArraySize(List<uint> values)
+        {
+            var countSize = GetUIntByteSize((uint)values.Count);
+            uint maxValue = 0;
+            for (var i = 0; i < values.Count; i++)
+            {
+                if (values[i] > maxValue)
+                {
+                    maxValue = values[i];
+                }
+            }
+
+            var entrySize = values.Count == 0 ? 1 : GetUIntByteSize(maxValue);
+            return 1 + countSize + 1 + values.Count * entrySize;
+        }
+
+        private static int GetUIntByteSize(uint value)
+        {
+            if (value <= 0xFF) return 1;
+            if (value <= 0xFFFF) return 2;
+            if (value <= 0xFFFFFF) return 3;
+            return 4;
         }
 
         private TrieNode GetNode(TrieNode node, char c, bool isEnd = false)

FreeMote/ZlibCompress.cs

@@ -1,5 +1,7 @@
-using System.IO;
+using System.IO;
 using System.IO.Compression;
+using ICSharpCode.SharpZipLib.Zip.Compression;
+using ICSharpCode.SharpZipLib.Zip.Compression.Streams;
 
 namespace FreeMote
 {
@@ -53,11 +55,23 @@ public static byte[] Compress(Stream input, bool fast = false)
             using var ms = new MemoryStream();
             ms.WriteByte(0x78);
             ms.WriteByte((byte) (fast ? 0x9C : 0xDA));
-            using (DeflateStream deflateStream =
-                   new DeflateStream(ms, Consts.ForceCompressionLevel != null ? Consts.ForceCompressionLevel.Value :
-                       fast ? CompressionLevel.Fastest : CompressionLevel.Optimal))
+            if (UseSharpZipLib(fast))
             {
-                input.CopyTo(deflateStream);
+                var deflaterLevel = GetDeflaterLevel(fast);
+                var deflater = new Deflater(deflaterLevel, true);
+                using (var deflateStream = new DeflaterOutputStream(ms, deflater))
+                {
+                    deflateStream.IsStreamOwner = false;
+                    input.CopyTo(deflateStream);
+                    deflateStream.Finish();
+                }
+            }
+            else
+            {
+                using (DeflateStream deflateStream = new DeflateStream(ms, GetSystemCompressionLevel(fast), true))
+                {
+                    input.CopyTo(deflateStream);
+                }
             }
 
             //input.Dispose();
@@ -81,17 +95,70 @@ public static MemoryStream CompressToStream(Stream input, bool fast = false)
             MemoryStream ms = new MemoryStream();
             ms.WriteByte(0x78);
             ms.WriteByte((byte) (fast ? 0x9C : 0xDA));
-            using (DeflateStream deflateStream =
-                   new DeflateStream(ms,
-                       Consts.ForceCompressionLevel != null ? Consts.ForceCompressionLevel.Value :
-                       fast ? CompressionLevel.Fastest : CompressionLevel.Optimal, true))
+            if (UseSharpZipLib(fast))
+            {
+                var deflaterLevel = GetDeflaterLevel(fast);
+                var deflater = new Deflater(deflaterLevel, true);
+                using (var deflateStream = new DeflaterOutputStream(ms, deflater))
+                {
+                    deflateStream.IsStreamOwner = false;
+                    input.CopyTo(deflateStream);
+                    deflateStream.Finish();
+                }
+            }
+            else
             {
-                input.CopyTo(deflateStream);
+                using (DeflateStream deflateStream = new DeflateStream(ms, GetSystemCompressionLevel(fast), true))
+                {
+                    input.CopyTo(deflateStream);
+                }
             }
 
             //input.Dispose(); //DO NOT dispose
             ms.Position = 0;
             return ms;
         }
+
+        private static int GetDeflaterLevel(bool fast)
+        {
+            if (Consts.ForceCompressionLevel != null)
+            {
+                switch (Consts.ForceCompressionLevel.Value)
+                {
+                    case CompressionLevel.NoCompression:
+                        return Deflater.NO_COMPRESSION;
+                    case CompressionLevel.Fastest:
+                        return Deflater.BEST_SPEED;
+                    case CompressionLevel.Optimal:
+                        return Deflater.BEST_COMPRESSION;
+                    default:
+                        return fast ? Deflater.BEST_SPEED : Deflater.BEST_COMPRESSION;
+                }
+            }
+
+            // MDF body in real games usually uses max deflate level (0xDA).
+            return fast ? Deflater.BEST_SPEED : Deflater.BEST_COMPRESSION;
+        }
+
+        private static CompressionLevel GetSystemCompressionLevel(bool fast)
+        {
+            if (Consts.ForceCompressionLevel != null)
+            {
+                return Consts.ForceCompressionLevel.Value;
+            }
+
+            return fast ? CompressionLevel.Fastest : CompressionLevel.Optimal;
+        }
+
+        private static bool UseSharpZipLib(bool fast)
+        {
+            if (fast)
+            {
+                return false;
+            }
+
+            // Only use SharpZipLib in optimize mode, so normal mode keeps .NET Deflate speed.
+            return Consts.OptimizeMode;
+        }
     }
 }