Skip to content

Commit 1107b38

Browse files
sk-zksk-zk
committed
Merge branch 'dev'
2 parents 1c43cd1 + cfa0a24 commit 1107b38

18 files changed

Lines changed: 458 additions & 128 deletions

‎Extractor.Tests/Deep/SiiPathFinderTest.cs‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,9 @@ public class SiiPathFinderTest
1313
[Fact]
1414
public void ProcessSiiSoundAttribute()
1515
{
16+
var spf = new SiiPathFinder();
1617
PotentialPaths actual = [];
17-
SiiPathFinder.ProcessSiiUnitAttribute("accessory_interior_data",
18+
spf.ProcessSiiUnitAttribute("accessory_interior_data",
1819
new("sounds", new string[] {
1920
"air_warning|/def/vehicle/truck/foo/air_warning.soundref",
2021
"system_warning1|/sound/truck/foo/bar.bank#interior/system_warning1",
@@ -25,7 +26,7 @@ public void ProcessSiiSoundAttribute()
2526
"/sound/truck/foo/bar.bank",
2627
"/sound/truck/foo/bar.bank.guids"
2728
];
28-
Assert.Equal(expected, actual);
29+
Assert.Subset(actual, expected);
2930
}
3031

3132
[Fact]
@@ -66,7 +67,8 @@ public void ConstructAccessoryIconPath()
6667
}
6768
""";
6869

69-
var (paths, _) = SiiPathFinder.FindPathsInSii(siiStr, "", null);
70+
var spf = new SiiPathFinder();
71+
var (paths, _) = spf.FindPathsInSii(siiStr, "", null);
7072

7173
Assert.Contains("/material/ui/accessory/hello/test.foo.mat", paths);
7274
}

‎Extractor/ConsoleUtils.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ public static void PrintRenameSummary(int renamed, int modified)
1616
if (renamed > 0)
1717
{
1818
Console.WriteLine($"WARN: {renamed} {(renamed == 1 ? "file" : "files")} had to be renamed and " +
19-
$"{modified} file{(renamed == 1 ? " was" : "s were")} modified accordingly.");
19+
$"{modified} file{(modified == 1 ? " was" : "s were")} modified accordingly.");
2020
}
2121
}
2222

‎Extractor/Deep/FilePathFinder.cs‎

Lines changed: 18 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
1-
using System;
1+
using Serilog;
2+
using System;
23
using System.Collections.Frozen;
34
using System.Collections.Generic;
45
using System.IO;
@@ -99,6 +100,10 @@ internal partial class FilePathFinder
99100

100101
private readonly HashSet<string> consumedSuis;
101102

103+
private readonly ILogger logger;
104+
105+
private readonly SiiPathFinder siiPathFinder = new();
106+
102107
/// <summary>
103108
/// Instantiates a <c>FilePathFinder</c>.
104109
/// </summary>
@@ -113,6 +118,8 @@ public FilePathFinder(IFileSystem fs, HashSet<string> visited,
113118
HashSet<string> referencedFiles, HashSet<string> dirsToSearchForRelativeTobj,
114119
HashSet<string> consumedSuis)
115120
{
121+
logger = Log.ForContext<FilePathFinder>();
122+
116123
this.visited = visited;
117124
this.dirsToSearchForRelativeTobj = dirsToSearchForRelativeTobj;
118125
this.referencedFiles = referencedFiles;
@@ -146,23 +153,9 @@ public PotentialPaths FindPathsInFile(byte[] fileBuffer, string filePath, FileTy
146153

147154
if (finders.TryGetValue(fileType, out var finder))
148155
{
149-
try
150-
{
151-
return finder(fileBuffer, filePath);
152-
}
153-
catch (Exception ex)
154-
{
155-
#if DEBUG
156-
Console.Error.WriteLine($"Unable to parse {filePath}: " +
157-
$"{ex.GetType().Name}: {ex.Message.Trim()}");
158-
#endif
159-
return [];
160-
}
161-
}
162-
else
163-
{
164-
return [];
156+
return finder(fileBuffer, filePath);
165157
}
158+
return [];
166159
}
167160

168161
/// <summary>
@@ -173,7 +166,7 @@ public PotentialPaths FindPathsInFile(byte[] fileBuffer, string filePath, FileTy
173166
/// <returns>Discovered paths.</returns>
174167
private PotentialPaths FindPathsInSii(byte[] fileBuffer, string filePath)
175168
{
176-
var (potentialPaths, newSuis) = SiiPathFinder.FindPathsInSii(fileBuffer, filePath, fs);
169+
var (potentialPaths, newSuis) = siiPathFinder.FindPathsInSii(fileBuffer, filePath, fs);
177170
consumedSuis.UnionWith(newSuis);
178171
potentialPaths.ExceptWith(visited);
179172
return potentialPaths;
@@ -211,19 +204,16 @@ private PotentialPaths FindPathsInFont(byte[] fileBuffer, string filePath)
211204
/// <returns>Discovered paths.</returns>
212205
private PotentialPaths FindPathsInMat(byte[] fileBuffer, string filePath)
213206
{
214-
PotentialPaths potentialPaths = [];
215-
216-
MatFile mat;
217-
try
218-
{
219-
mat = MatFile.Load(Encoding.UTF8.GetString(fileBuffer));
220-
}
221-
catch (Exception)
207+
// Purposefully mislabeled .sii file?
208+
if (fileBuffer.Length > 8 && fileBuffer.AsSpan(0, 8).SequenceEqual("SiiNunit"u8))
222209
{
223-
//Debugger.Break();
224-
throw;
210+
return FindPathsInSii(fileBuffer, filePath);
225211
}
226212

213+
PotentialPaths potentialPaths = [];
214+
215+
var mat = MatFile.Load(Encoding.UTF8.GetString(fileBuffer));
216+
227217
foreach (var texture in mat.Textures)
228218
{
229219
if (!texture.Attributes.TryGetValue("source", out var value)

‎Extractor/Deep/HashFsDeepExtractor.cs‎

Lines changed: 43 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,16 @@
1-
using Sprache;
1+
using Serilog;
2+
using Sprache;
23
using System;
34
using System.Collections.Generic;
5+
using System.Data;
46
using System.Diagnostics;
57
using System.IO;
68
using System.Linq;
79
using System.Text;
810
using System.Threading.Tasks;
9-
using System.Data;
1011
using TruckLib.HashFs;
11-
using static Extractor.PathUtils;
1212
using static Extractor.ConsoleUtils;
13+
using static Extractor.PathUtils;
1314
using static Extractor.TextUtils;
1415

1516
namespace Extractor.Deep
@@ -41,8 +42,11 @@ public class HashFsDeepExtractor : HashFsExtractor
4142

4243
private bool hasSearchedForPaths;
4344

45+
private readonly ILogger logger;
46+
4447
public HashFsDeepExtractor(string scsPath, Options opt) : base(scsPath, opt)
4548
{
49+
logger = Log.ForContext<HashFsDeepExtractor>();
4650
}
4751

4852
/// <inheritdoc/>
@@ -58,6 +62,7 @@ public void Extract(IList<string> foundFiles, string outputRoot, bool ignoreMiss
5862
bool startPathsSet = !opt.StartPaths.SequenceEqual(["/"]);
5963

6064
substitutions = DeterminePathSubstitutions(foundFiles);
65+
LogPathSubstitutions();
6166

6267
// Extract regular files
6368
var filteredFoundFiles = foundFiles
@@ -69,6 +74,8 @@ public void Extract(IList<string> foundFiles, string outputRoot, bool ignoreMiss
6974
})
7075
.Order()
7176
.ToArray();
77+
logger.Information("[{ScsName}] Extracting {NumPaths} files with known paths",
78+
ScsName, filteredFoundFiles.Length);
7279
ExtractFiles(filteredFoundFiles, outputRoot, ignoreMissing);
7380

7481
// Extract decoy files
@@ -81,6 +88,11 @@ public void Extract(IList<string> foundFiles, string outputRoot, bool ignoreMiss
8188
})
8289
.Order()
8390
.ToArray();
91+
if (foundDecoyFiles.Length > 0)
92+
{
93+
logger.Information("[{ScsName}] Extracting {NumDecoyPaths} decoy files",
94+
ScsName, foundDecoyFiles.Length);
95+
}
8496
var decoyDestination = Path.Combine(outputRoot, DecoyDirectory);
8597
foreach (var decoyFile in foundDecoyFiles)
8698
{
@@ -99,12 +111,15 @@ public void Extract(IList<string> foundFiles, string outputRoot, bool ignoreMiss
99111

100112
public (HashSet<string> FoundFiles, HashSet<string> ReferencedFiles) FindPaths()
101113
{
114+
logger.Information("[{ScsName}] Searching archive contents for paths", ScsName);
102115
if (!hasSearchedForPaths)
103116
{
104117
finder = new HashFsPathFinder(Reader, opt.AdditionalStartPaths, junkEntries);
105118
finder.Find();
106119
hasSearchedForPaths = true;
107120
}
121+
logger.Information("[{ScsName}] Found {NumFoundFiles} paths, {NumReferencedFiles} referenced paths",
122+
ScsName, finder.FoundFiles.Count, finder.ReferencedFiles.Count);
108123
return (finder.FoundFiles, finder.ReferencedFiles);
109124
}
110125

@@ -133,21 +148,31 @@ private void DumpUnrecovered(string destination, IEnumerable<string> foundFiles)
133148
}
134149
junkEntries.TryGetValue(Reader.HashPath(f), out var retval);
135150
return retval;
136-
}));
151+
}))
152+
.Except(junkEntries.Values)
153+
.Except(maybeJunkEntries.Values)
154+
.ToArray();
137155

156+
if (notRecovered.Length == 0)
157+
{
158+
return;
159+
}
160+
161+
logger.Information("[{ScsName}] Dumping {NumUnrecovered} files with unknown paths",
162+
ScsName, notRecovered.Length);
138163
HashSet<ulong> visitedOffsets = [];
139164

140165
var outputDir = Path.Combine(destination, DumpDirectory);
141166

142167
foreach (var entry in notRecovered)
143168
{
144-
if (junkEntries.ContainsKey(entry.Hash) ||
169+
/*if (junkEntries.ContainsKey(entry.Hash) ||
145170
maybeJunkEntries.ContainsKey(entry.Hash))
146171
{
147172
continue;
148-
}
173+
}*/
149174

150-
var offsetHexStr = entry.Hash.ToString("x16");
175+
var hashHexStr = entry.Hash.ToString("x16");
151176

152177
byte[] fileBuffer;
153178
try
@@ -156,14 +181,15 @@ private void DumpUnrecovered(string destination, IEnumerable<string> foundFiles)
156181
}
157182
catch (Exception ex)
158183
{
159-
Console.WriteLine($"Uanble to dump {offsetHexStr}: {ex.Message}");
184+
Console.WriteLine($"Unable to dump {hashHexStr}: {ex.Message}");
185+
logger.Error(ex, "[{ScsName}] Unable to dump {HashStr} ({Offset})", ScsName, hashHexStr, entry.Offset);
160186
numFailed++;
161187
continue;
162188
}
163189

164190
var fileType = FileTypeHelper.Infer(fileBuffer);
165191
var extension = FileTypeHelper.FileTypeToExtension(fileType);
166-
var fileName = offsetHexStr + extension;
192+
var fileName = hashHexStr + extension;
167193
var outputPath = Path.Combine(outputDir, fileName);
168194
Console.WriteLine($"Dumping {fileName} ...");
169195
if (!Overwrite && File.Exists(outputPath))
@@ -172,6 +198,7 @@ private void DumpUnrecovered(string destination, IEnumerable<string> foundFiles)
172198
}
173199
else
174200
{
201+
logger.Verbose("[{ScsName}] Dumping {FileName}", ScsName, fileName);
175202
ExtractToDisk(entry, $"/{DumpDirectory}/{fileName}", outputPath);
176203
numDumped++;
177204
}
@@ -180,6 +207,7 @@ private void DumpUnrecovered(string destination, IEnumerable<string> foundFiles)
180207

181208
public override void PrintPaths(bool includeAll)
182209
{
210+
logger.Information("[{ScsName}] Searching archive contents for paths", ScsName);
183211
var finder = new HashFsPathFinder(Reader);
184212
finder.Find();
185213
var paths = (includeAll
@@ -194,11 +222,17 @@ public override void PrintExtractionResult()
194222
Console.WriteLine($"{numExtracted} extracted " +
195223
$"({renamedFiles.Count} renamed, {modifiedFiles.Count} modified, {numDumped} dumped), " +
196224
$"{numSkipped} skipped, {numJunk} junk, {numFailed} failed");
225+
logger.Information("[{ScsName}] {NumExtracted} extracted " +
226+
"({NumRenamed} renamed, {NumModified} modified, {NumDumped} dumped), " +
227+
"{NumSkipped} skipped, {NumJunk} junk, {NumFailed} failed",
228+
ScsName, numExtracted, renamedFiles.Count, modifiedFiles.Count,
229+
numDumped, numSkipped, numJunk, numFailed);
197230
PrintRenameSummary(renamedFiles.Count, modifiedFiles.Count);
198231
}
199232

200233
public override List<Tree.Directory> GetDirectoryTree()
201234
{
235+
logger.Information("[{ScsName}] Searching archive contents for paths", ScsName);
202236
var finder = new HashFsPathFinder(Reader);
203237
finder.Find();
204238

0 commit comments

Comments
 (0)