Skip to content

Commit 5f51e74

Browse files
henbaglehenbagle
committed
Implement new TOC hashing algorithm, part of #1 
1 parent cd417e3 commit 5f51e74

4 files changed

Lines changed: 178 additions & 123 deletions

File tree

‎AutoTOC/Program.cs‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -78,13 +78,16 @@ static void GenerateTocFromGamedir(string gameDir, MEGame game)
7878
string dlcDir = Path.Combine(baseDir, @"DLC\");
7979
List<string> folders = new List<string>();
8080
folders.Add(baseDir);
81-
if(Directory.Exists(dlcDir))
81+
if (game != MEGame.LE1)
8282
{
83-
folders.AddRange((new DirectoryInfo(dlcDir)).GetDirectories().Select(d => d.FullName));
84-
}
85-
else
86-
{
87-
Console.WriteLine("DLC folder not detected, TOCing basegame only...");
83+
if(Directory.Exists(dlcDir))
84+
{
85+
folders.AddRange((new DirectoryInfo(dlcDir)).GetDirectories().Select(d => d.FullName));
86+
}
87+
else
88+
{
89+
Console.WriteLine("DLC folder not detected, TOCing basegame only...");
90+
}
8891
}
8992
Task.WhenAll(folders.Select(loc => TOCAsync(loc, game))).Wait();
9093
}

‎AutoTOC/Properties/AssemblyInfo.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,5 +32,5 @@
3232
// You can specify all the values or you can default the Build and Revision Numbers
3333
// by using the '*' as shown below:
3434
// [assembly: AssemblyVersion("1.0.*")]
35-
[assembly: AssemblyVersion("2.1.*")]
36-
[assembly: AssemblyFileVersion("2.1.1.0")]
35+
[assembly: AssemblyVersion("2.2.*")]
36+
[assembly: AssemblyFileVersion("2.2.0.0")]

‎AutoTOC/TOCBinFile.cs‎

Lines changed: 51 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -117,60 +117,59 @@ public class TOCHashTableEntry
117117
{
118118
internal int offset { get; set; }
119119
internal int entrycount { get; set; }
120-
internal List<Entry> TOCEntries { get; } = new List<Entry>();
120+
public List<Entry> TOCEntries { get; } = new List<Entry>();
121121
}
122122

123-
/*
124-
public void ReadFile(MemoryStream ms)
125-
{
126-
var reader = new EndianReader(ms);
127-
uint magic = (uint)reader.ReadInt32();
128-
if (magic != TOCMagicNumber)
129-
{
130-
throw new Exception("Not a TOC file, bad magic header");
131-
}
132-
133-
var mediaTableCount = reader.ReadInt32(); // Should be 0
134-
var hashTableCount = reader.ReadInt32();
135-
136-
long maxReadValue = 0;
137-
for (int i = 0; i < hashTableCount; i++)
138-
{
139-
var pos = reader.Position;
140-
var newEntry = new TOCHashTableEntry()
141-
{
142-
offset = reader.ReadInt32(),
143-
entrycount = reader.ReadInt32(),
144-
};
145-
HashBuckets.Add(newEntry);
146-
147-
var resumePosition = reader.Position;
148-
// Read Entries
149-
150-
reader.Position = newEntry.offset + pos;
151-
for (int j = 0; j < newEntry.entrycount; j++)
152-
{
153-
154-
Entry e = new()
155-
{
156-
offset = (int)reader.Position,
157-
entrydisksize = reader.ReadInt16(),
158-
flags = reader.ReadInt16(),
159-
size = reader.ReadInt32(),
160-
sha1 = reader.ReadToBuffer(0x14), // 20
161-
name = reader.ReadStringASCIINull()
162-
};
163-
164-
reader.Seek(e.offset + e.entrydisksize, SeekOrigin.Begin);
165-
166-
maxReadValue = Math.Max(maxReadValue, reader.Position);
167-
168-
newEntry.TOCEntries.Add(e);
169-
}
170-
171-
reader.Position = resumePosition;
172-
}
173-
} */
123+
//public void ReadFile(MemoryStream ms)
124+
//{
125+
// var reader = new EndianReader(ms);
126+
// uint magic = (uint)reader.ReadInt32();
127+
// if (magic != TOCMagicNumber)
128+
// {
129+
// throw new Exception("Not a TOC file, bad magic header");
130+
// }
131+
132+
// var mediaTableCount = reader.ReadInt32(); // Should be 0
133+
// var hashTableCount = reader.ReadInt32();
134+
135+
// long maxReadValue = 0;
136+
// for (int i = 0; i < hashTableCount; i++)
137+
// {
138+
// var pos = reader.Position;
139+
// var newEntry = new TOCHashTableEntry()
140+
// {
141+
// offset = reader.ReadInt32(),
142+
// entrycount = reader.ReadInt32(),
143+
// };
144+
// HashBuckets.Add(newEntry);
145+
146+
// var resumePosition = reader.Position;
147+
// // Read Entries
148+
149+
// reader.Position = newEntry.offset + pos;
150+
// for (int j = 0; j < newEntry.entrycount; j++)
151+
// {
152+
153+
// Entry e = new()
154+
// {
155+
// offset = (int)reader.Position,
156+
// entrydisksize = reader.ReadInt16(),
157+
// flags = reader.ReadInt16(),
158+
// size = reader.ReadInt32(),
159+
// sha1 = reader.ReadToBuffer(0x14), // 20
160+
// name = reader.ReadStringASCIINull()
161+
// };
162+
163+
// reader.Seek(e.offset + e.entrydisksize, SeekOrigin.Begin);
164+
165+
// maxReadValue = Math.Max(maxReadValue, reader.Position);
166+
167+
// newEntry.TOCEntries.Add(e);
168+
// }
169+
170+
// reader.Position = resumePosition;
171+
// }
172+
//}
174173

175174

176175
public void UpdateEntry(int Index, int size)

‎AutoTOC/TOCCreator.cs‎

Lines changed: 116 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -45,14 +45,14 @@ public static IEnumerable<string> GetTocableFiles(string path)
4545
private static List<string> GetFiles(string basefolder, bool isLE2LE3)
4646
{
4747
var res = new List<string>();
48-
string directoryName = Path.GetFileName(Path.GetDirectoryName(basefolder));
48+
string directoryName = Path.GetFileName(basefolder);
4949
// Do not include the directory's existing PCConsoleTOC.bin
5050
res.AddRange(GetTocableFiles(basefolder).Except(new[] { Path.Combine(basefolder, "PCConsoleTOC.bin") }, StringComparer.InvariantCultureIgnoreCase));
5151
DirectoryInfo folder = new DirectoryInfo(basefolder);
5252
var folders = folder.GetDirectories();
5353
if (folders.Length != 0)
5454
{
55-
if (!directoryName.Equals("BioGame", StringComparison.InvariantCultureIgnoreCase))
55+
if (!folder.Name.Equals("BioGame", StringComparison.InvariantCultureIgnoreCase))
5656
{
5757
//treat as dlc and include all folders.
5858
foreach (DirectoryInfo f in folders)
@@ -63,10 +63,8 @@ private static List<string> GetFiles(string basefolder, bool isLE2LE3)
6363
//biogame, only do cookedpcconsole and movies.
6464
foreach (DirectoryInfo f in folders)
6565
{
66-
if (f.Name == "CookedPCConsole" || f.Name == "Movies")
66+
if (f.Name == "CookedPCConsole" || f.Name == "Movies" || f.Name == "DLC")
6767
res.AddRange(GetFiles(Path.Combine(basefolder, f.Name), isLE2LE3));
68-
else if (isLE2LE3 && f.Name == "DLC")
69-
res.AddRange(GetFiles(Path.Combine(basefolder, f.Name), isLE2LE3)); // may need updated when we get LE1 DLC system up
7068
else if (f.Name == "Content")
7169
res.AddRange(GetFiles(Path.Combine(basefolder, f.Name, "Packages", "ISACT"), isLE2LE3));
7270

@@ -94,7 +92,7 @@ public static MemoryStream CreateTOCForDirectory(string directory, MEGame game)
9492
//Strip the non-relative path information
9593
string file0fullpath = files[0];
9694
int dlcFolderStartSubStrPos = file0fullpath.IndexOf("DLC_", StringComparison.InvariantCultureIgnoreCase);
97-
if (dlcFolderStartSubStrPos > 0)
95+
if (dlcFolderStartSubStrPos > 0 && game != MEGame.LE1)
9896
{
9997
// DLC TOC
10098
files = files.Select(x => x.Substring(dlcFolderStartSubStrPos)).ToList();
@@ -134,88 +132,143 @@ public static MemoryStream CreateTOCForDirectory(string directory, MEGame game)
134132
/// </summary>
135133
/// <param name="filesystemInfo">list of filenames and sizes for the TOC</param>
136134
/// <returns>memorystream of TOC, null if list is empty</returns>
137-
public static MemoryStream CreateTOCForEntries(List<(string filename, int size)> filesystemInfo)
135+
public static MemoryStream CreateTOCForEntries(List<(string relativeFilename, int size)> filesystemInfo)
138136
{
139137
if (filesystemInfo.Count != 0)
140138
{
141139
var tbf = new TOCBinFile();
142140

143-
// Todo: Update this someday so it lines up with the actual correct implementation
144-
var hashBucket = new TOCBinFile.TOCHashTableEntry();
145-
tbf.HashBuckets.Add(hashBucket);
146-
hashBucket.TOCEntries.AddRange(filesystemInfo.Select(x => new TOCBinFile.Entry
147-
{
148-
flags = 0,
149-
name = x.filename,
150-
size = x.size
151-
}));
152-
153-
return tbf.Save();
154-
}
141+
// Generate hashes for all names
142+
Dictionary<(string relativeFileName, int size), uint> fullHashMap =
143+
new Dictionary<(string relativeFileName, int size), uint>(); // Unbounded hash table size
155144

156-
return null;
157-
/*
158-
MemoryStream fs = MemoryManager.GetMemoryStream();
159-
160-
fs.WriteInt32(TOCBinFile.TOCMagicNumber); // Endian check
161-
fs.WriteInt32(0x0); // Media Data Count
162-
fs.WriteInt32(0x1); // Hash Table Count
145+
foreach (var f in filesystemInfo)
146+
{
147+
fullHashMap[f] = GetStringFullHash(Path.GetFileName(f.relativeFilename));
148+
}
163149

164-
// TOCHashTableEntry (Only have 1 entry)
165-
fs.WriteInt32(0x8); // Offset of first entry from
166-
fs.WriteInt32(filesystemInfo.Count); // Number of files in this table
150+
// Calculate optimal hash table size for performance
151+
var hashTableSize = filesystemInfo.Count; // Initial size is 100% 1:1
152+
List<TOCBinFile.TOCHashTableEntry> hashBuckets;
167153

168-
for (int i = 0; i < filesystemInfo.Count; i++)
154+
while (true)
169155
{
156+
hashBuckets = new List<TOCBinFile.TOCHashTableEntry>();
157+
for(int i = 0; i < hashTableSize; i++) hashBuckets.Add(new TOCBinFile.TOCHashTableEntry());
170158

171-
// TOCFileEntry - 4 Byte Aligned
172-
(string file, int size) entry = filesystemInfo[i];
159+
//Populate the buckets with file entries
160+
foreach (var hashPair in fullHashMap)
161+
{
162+
var bucketIdx = GetBoundedHashValue(hashPair.Value, hashTableSize);
163+
hashBuckets[(int)bucketIdx].TOCEntries.Add(new TOCBinFile.Entry()
164+
{
165+
flags = 0,
166+
name = hashPair.Key.relativeFileName,
167+
size = hashPair.Key.size
168+
});
169+
}
173170

174-
// Next Entry Offset
175-
if (i == filesystemInfo.Count - 1) // Last entry has no next offset
176-
fs.WriteAligned(BitConverter.GetBytes((ushort)0), 0, 2, 4);
177-
else
178-
fs.WriteAligned(BitConverter.GetBytes((ushort)entry.file.Length), 0, 2, 4);
179171

180-
//nextEntryOffsetPos = fs.Position;
181-
fs.WriteUInt16((ushort)(0x1D + entry.file.Length)); // Next entry start offset
172+
// Check fill rate. We must have over 75% fill rate or we will lower the hash table size by 25% and try again (down to 50% of file table size)
173+
var emptyHashBucketsCount = hashBuckets.Count(x => x.TOCEntries.Count == 0);
182174

183-
// Flags
184-
fs.WriteUInt16(0);
175+
//Console.WriteLine($@"Hash fill rate: {100 - (emptyHashBucketsCount * 100.0f / hashTableSize)}%");
185176

186-
// FileSize
187-
if (!Path.GetFileName(entry.file).Equals("PCConsoleTOC.bin", StringComparison.InvariantCultureIgnoreCase))
177+
if (emptyHashBucketsCount > hashTableSize / 4)
188178
{
189-
fs.WriteInt32(entry.size);
179+
int shrunkTableSize = Math.Max(filesystemInfo.Count / 2, hashTableSize - (hashTableSize / 4));
180+
if (shrunkTableSize == hashTableSize)
181+
{
182+
// WARNING: This will be suboptimal hash table size but we are going to use it anyways to prevent crash
183+
Console.WriteLine(@"WARNING: Hash table fill rate is low; even at 50% of the file size");
184+
Console.WriteLine(@"This TOC file will work, but is suboptimal.");
185+
break;
186+
}
187+
188+
// Update size
189+
hashTableSize = shrunkTableSize;
190+
continue;
190191
}
191192
else
192193
{
193-
selfSizePosition = fs.Position; // Save self-position so we can rewrite our own file size at the end
194-
fs.WriteInt32(0);
194+
// This is a satisfactory TOC
195+
break;
195196
}
197+
}
196198

197-
// SHA1 - Not used by games...
198-
fs.WriteZeros(20);
199-
fs.WriteStringLatin1(entry.file); // Not present in hash table?
199+
tbf.HashBuckets = hashBuckets;
200+
return tbf.Save();
201+
}
200202

201-
// Old method
202-
//foreach (char c in file)
203-
// fs.WriteByte((byte)c);
204-
fs.WriteByte(0);
205-
}
203+
return null;
204+
}
206205

207-
if (selfSizePosition >= 0)
208-
{
209-
// Write the size of our own TOC. This ensures TOC appears up to date when we try to update it later
210-
// (important for DLC TOCs)
211-
fs.Seek(selfSizePosition, SeekOrigin.Begin);
212-
fs.WriteInt32((int)fs.Length);
213-
}
206+
/// <summary>
207+
/// Gets the hash value of a string without bounding (UE3-hash)
208+
/// </summary>
209+
/// <param name="strToHash"></param>
210+
/// <returns></returns>
211+
private static uint GetStringFullHash(string strToHash)
212+
{
213+
initCRCTable();
214214

215-
return fs;
215+
uint hash = 0;
216+
var upperCaseStr = strToHash.ToUpper();
217+
for (var i = 0; i < upperCaseStr.Length; ++i)
218+
{
219+
char upperChar = upperCaseStr[i];
220+
hash = ((hash >> 8) & 0x00FFFFFF) ^ crcTable[(hash ^ ((byte)upperChar)) & 0x000000FF]; // ASCII
221+
hash = ((hash >> 8) & 0x00FFFFFF) ^ crcTable[(hash) & 0x000000FF]; // This is for unicode as each character is two bytes.
216222
}
223+
return hash;
224+
}
225+
226+
/// <summary>
227+
/// Gets the hash value of a string with bounding (UE3-hash)
228+
/// </summary>
229+
/// <param name="inputString"></param>
230+
/// <param name="hashTableSize"></param>
231+
/// <returns></returns>
232+
private static uint GetStringHashBounded(string inputString, int hashTableSize)
233+
{
234+
return (uint)(GetStringFullHash(inputString) % hashTableSize);
235+
}
236+
237+
/// <summary>
238+
/// Applies the specified bound to the listed hash
239+
/// </summary>
240+
/// <param name="hash"></param>
241+
/// <param name="bound"></param>
242+
/// <returns></returns>
243+
private static uint GetBoundedHashValue(uint hash, int bound)
244+
{
245+
return (uint)(hash % bound);
246+
}
247+
248+
private static uint[] crcTable;
249+
250+
/// <summary>
251+
/// Polynomial for our CRCs
252+
/// </summary>
253+
private const uint CRC_POLYNOMIAL = 0x04C11DB7;
217254

218-
return null;*/
255+
/// <summary>
256+
/// Initializes the CRC table which is used for calculating a hash
257+
/// </summary>
258+
private static void initCRCTable()
259+
{
260+
if (crcTable != null) return;
261+
crcTable = new uint[256];
262+
// Table has 256 entries.
263+
for (uint idx = 0; idx < 256; idx++)
264+
{
265+
// Generate CRCs based on the polynomial
266+
for (uint crc = idx << 24, bitIdx = 8; bitIdx != 0; bitIdx--)
267+
{
268+
crc = ((crc & 0x80000000) == 0x80000000) ? (crc << 1) ^ CRC_POLYNOMIAL : crc << 1;
269+
crcTable[idx] = crc;
270+
}
271+
}
219272
}
220273
}
221274
}

0 commit comments

Comments
 (0)