Skip to content

Commit bba209a

Browse files
ihatecompvirihatecompvir
committed
More parallelization wins; take advantage of SIMD for speedups
opening and saving every RB3 Milo now takes 20 seconds less, massive W
1 parent fd89125 commit bba209a

6 files changed

Lines changed: 256 additions & 136 deletions

File tree

‎ImMilo/CharAssetFixer.cs‎

Lines changed: 16 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -98,13 +98,18 @@ public static void FixCharAssetFolder(string path)
9898
var files = Directory.GetFiles(path);
9999
var fixedDir = Path.Join(path, "fixed");
100100
Directory.CreateDirectory(fixedDir);
101-
foreach (var filePath in files)
101+
int progress = 0;
102+
Parallel.ForEach(files, filePath =>
102103
{
103-
Console.WriteLine($"Fixing {filePath}");
104+
var count = Interlocked.Increment(ref progress);
105+
lock (Console.Out)
106+
{
107+
Console.WriteLine($"({count}/{files.Length}) Fixing {filePath}");
108+
}
104109
var filename = Path.GetFileName(filePath);
105110
var newPath = Path.Join(fixedDir, filename);
106111
FixCharAsset(new MiloFile(filePath), newPath);
107-
}
112+
});
108113
}
109114

110115
public static void PromptCharAssetFix()
@@ -201,13 +206,18 @@ public static void FixInstrumentFolder(string path)
201206
var files = Directory.GetFiles(path);
202207
var fixedDir = Path.Join(path, "fixed");
203208
Directory.CreateDirectory(fixedDir);
204-
foreach (var filePath in files)
209+
int progress = 0;
210+
Parallel.ForEach(files, filePath =>
205211
{
206-
Console.WriteLine($"Fixing {filePath}");
212+
var count = Interlocked.Increment(ref progress);
213+
lock (Console.Out)
214+
{
215+
Console.WriteLine($"({count}/{files.Length}) Fixing {filePath}");
216+
}
207217
var filename = Path.GetFileName(filePath);
208218
var newPath = Path.Join(fixedDir, filename);
209219
FixInstrument(new MiloFile(filePath), newPath);
210-
}
220+
});
211221
}
212222

213223
public static void PromptInstrumentFix()

‎MiloLib/Assets/DirectoryMeta.cs‎

Lines changed: 66 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -940,32 +940,53 @@ public DirectoryMeta Read(EndianReader reader)
940940
this.DirObjBytes = reader.ReadBlock((int)(dirDataEnd - dirDataStart));
941941
}
942942

943+
// HMX's DEADDEAD magic value used to signify an object has ended
944+
ReadOnlySpan<byte> endMarker = stackalloc byte[] { 0xAD, 0xDE, 0xAD, 0xDE };
945+
943946
foreach (Entry entry in entries)
944947
{
945948
long startPos = reader.BaseStream.Position;
946949
entry.objBytesAbsolutePosition = startPos;
947950

948-
var objBytesList = new List<byte>();
949-
while (true)
951+
// SIMD go brrrrrrrrrrr
952+
if (reader.BaseStream is MemoryStream ms)
950953
{
951-
byte b = reader.ReadByte();
952-
if (b == 0xAD)
954+
byte[] buffer = ms.GetBuffer();
955+
int pos = (int)ms.Position;
956+
int end = (int)ms.Length;
957+
int idx = buffer.AsSpan(pos, end - pos).IndexOf(endMarker);
958+
if (idx >= 0)
953959
{
954-
long currentPos = reader.BaseStream.Position;
955-
956-
if (reader.ReadByte() == 0xDE &&
957-
reader.ReadByte() == 0xAD &&
958-
reader.ReadByte() == 0xDE)
960+
entry.objBytes = new byte[idx];
961+
Buffer.BlockCopy(buffer, pos, entry.objBytes, 0, idx);
962+
}
963+
else
964+
{
965+
entry.objBytes = Array.Empty<byte>();
966+
}
967+
}
968+
else
969+
{
970+
// fall back to the old byte-by-byte method if needed
971+
var objBytesList = new List<byte>();
972+
while (true)
973+
{
974+
byte b = reader.ReadByte();
975+
if (b == 0xAD)
959976
{
960-
break;
977+
long currentPos = reader.BaseStream.Position;
978+
if (reader.ReadByte() == 0xDE &&
979+
reader.ReadByte() == 0xAD &&
980+
reader.ReadByte() == 0xDE)
981+
{
982+
break;
983+
}
984+
reader.BaseStream.Position = currentPos;
961985
}
962-
963-
reader.BaseStream.Position = currentPos;
986+
objBytesList.Add(b);
964987
}
965-
966-
objBytesList.Add(b);
988+
entry.objBytes = objBytesList.ToArray();
967989
}
968-
entry.objBytes = objBytesList.ToArray();
969990

970991
reader.BaseStream.Position = startPos;
971992

@@ -1054,22 +1075,41 @@ public void ReadEntry(EndianReader reader, DirectoryMeta.Entry entry)
10541075

10551076
entry.typeRecognized = false;
10561077

1057-
// TODO: improve this shit
1058-
while (true)
1078+
// again, try to use SIMD accel when possible
1079+
ReadOnlySpan<byte> unknownEndMarker = stackalloc byte[] { 0xAD, 0xDE, 0xAD, 0xDE };
1080+
if (reader.BaseStream is MemoryStream ums)
10591081
{
1060-
byte b = reader.ReadByte();
1061-
if (b == 0xAD)
1082+
byte[] buffer = ums.GetBuffer();
1083+
int pos = (int)ums.Position;
1084+
int end = (int)ums.Length;
1085+
int idx = buffer.AsSpan(pos, end - pos).IndexOf(unknownEndMarker);
1086+
if (idx >= 0)
10621087
{
1063-
byte b2 = reader.ReadByte();
1064-
if (b2 == 0xDE)
1088+
ums.Position = pos + idx + 4; // skip past end marker
1089+
}
1090+
else
1091+
{
1092+
throw new EndOfStreamException("Could not find end marker for unknown entry type " + entry.type.value);
1093+
}
1094+
}
1095+
else
1096+
{
1097+
while (true)
1098+
{
1099+
byte b = reader.ReadByte();
1100+
if (b == 0xAD)
10651101
{
1066-
byte b3 = reader.ReadByte();
1067-
if (b3 == 0xAD)
1102+
byte b2 = reader.ReadByte();
1103+
if (b2 == 0xDE)
10681104
{
1069-
byte b4 = reader.ReadByte();
1070-
if (b4 == 0xDE)
1105+
byte b3 = reader.ReadByte();
1106+
if (b3 == 0xAD)
10711107
{
1072-
break;
1108+
byte b4 = reader.ReadByte();
1109+
if (b4 == 0xDE)
1110+
{
1111+
break;
1112+
}
10731113
}
10741114
}
10751115
}

0 commit comments

Comments
 (0)