Skip to content

Commit e055229

Browse files
Compute typemap fingerprints in a single streaming pass
The content fingerprint (which seeds the deterministic MVID) and the incremental-build fingerprint each walked the whole model into a BinaryWriter over a MemoryStream, then hashed the buffer. On a real Mono.Android workload those two buffered walks were the single largest allocation source in the generator. Serialise both fingerprints from one walk, streaming the fields straight into SHA-256 through a small reusable buffer. Fields shared by the two fingerprints are UTF-8 encoded once and appended to both hashes, so the byte stream seen by each hash — and therefore every fingerprint value and generated MVID — is unchanged. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
1 parent e8ac902 commit e055229

6 files changed

Lines changed: 735 additions & 173 deletions

File tree

Lines changed: 174 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,174 @@
1+
using System;
2+
using System.Security.Cryptography;
3+
using System.Text;
4+
5+
namespace Microsoft.Android.Sdk.TrimmableTypeMap;
6+
7+
/// <summary>
8+
/// Streams model fields into one or two SHA-256 hashes without materialising the whole
9+
/// serialised model in memory.
10+
/// </summary>
11+
/// <remarks>
12+
/// <para>
13+
/// The byte stream produced for each sink is identical to what
14+
/// <see cref="System.IO.BinaryWriter"/> would have written into a <see cref="System.IO.MemoryStream"/>:
15+
/// strings are UTF-8 encoded with a 7-bit encoded byte-length prefix, <see cref="bool"/> is one
16+
/// byte, and <see cref="int"/> is four bytes little-endian. Keeping the encoding identical means
17+
/// the resulting fingerprints — and therefore the deterministic MVIDs derived from them — are
18+
/// unchanged.
19+
/// </para>
20+
/// <para>
21+
/// Two sinks are supported so the content fingerprint (which seeds the MVID) and the
22+
/// incremental-build fingerprint can be produced from a single walk over the model. Fields shared
23+
/// by both fingerprints are UTF-8 encoded once and appended to both sinks; fields belonging to
24+
/// only one fingerprint are appended to that sink alone.
25+
/// </para>
26+
/// </remarks>
27+
sealed class FingerprintWriter : IDisposable
28+
{
29+
/// <summary>Selects which fingerprint(s) a write applies to.</summary>
30+
[Flags]
31+
public enum Sink
32+
{
33+
Content = 1,
34+
Incremental = 2,
35+
Both = Content | Incremental,
36+
}
37+
38+
// Large enough to absorb the small field writes that dominate the model walk without
39+
// paying per-field hash update costs, small enough to stay off the large object heap.
40+
const int BufferSize = 8 * 1024;
41+
42+
readonly IncrementalHash contentHash;
43+
readonly IncrementalHash? incrementalHash;
44+
readonly byte [] contentBuffer;
45+
readonly byte []? incrementalBuffer;
46+
byte [] scratch = new byte [512];
47+
int contentPosition;
48+
int incrementalPosition;
49+
50+
public FingerprintWriter (bool includeIncremental)
51+
{
52+
contentHash = IncrementalHash.CreateHash (HashAlgorithmName.SHA256);
53+
contentBuffer = new byte [BufferSize];
54+
if (includeIncremental) {
55+
incrementalHash = IncrementalHash.CreateHash (HashAlgorithmName.SHA256);
56+
incrementalBuffer = new byte [BufferSize];
57+
}
58+
}
59+
60+
public void WriteString (Sink sink, string value)
61+
{
62+
int byteCount = Encoding.UTF8.GetByteCount (value);
63+
EnsureScratch (byteCount + 5);
64+
int offset = Write7BitEncodedInt (scratch, 0, byteCount);
65+
Encoding.UTF8.GetBytes (value, 0, value.Length, scratch, offset);
66+
Write (sink, scratch, 0, offset + byteCount);
67+
}
68+
69+
public void WriteOptionalString (Sink sink, string? value)
70+
{
71+
WriteBoolean (sink, value is not null);
72+
if (value is not null) {
73+
WriteString (sink, value);
74+
}
75+
}
76+
77+
public void WriteBoolean (Sink sink, bool value) => WriteByte (sink, value ? (byte) 1 : (byte) 0);
78+
79+
public void WriteByte (Sink sink, byte value)
80+
{
81+
scratch [0] = value;
82+
Write (sink, scratch, 0, 1);
83+
}
84+
85+
public void WriteInt32 (Sink sink, int value)
86+
{
87+
scratch [0] = (byte) value;
88+
scratch [1] = (byte) (value >> 8);
89+
scratch [2] = (byte) (value >> 16);
90+
scratch [3] = (byte) (value >> 24);
91+
Write (sink, scratch, 0, 4);
92+
}
93+
94+
public void WriteRaw (Sink sink, byte [] value) => Write (sink, value, 0, value.Length);
95+
96+
public byte [] GetContentFingerprint ()
97+
{
98+
FlushContent ();
99+
return contentHash.GetHashAndReset ();
100+
}
101+
102+
public byte [] GetIncrementalFingerprint ()
103+
{
104+
if (incrementalHash is null || incrementalBuffer is null) {
105+
throw new InvalidOperationException ("The incremental fingerprint was not requested.");
106+
}
107+
if (incrementalPosition > 0) {
108+
incrementalHash.AppendData (incrementalBuffer, 0, incrementalPosition);
109+
incrementalPosition = 0;
110+
}
111+
return incrementalHash.GetHashAndReset ();
112+
}
113+
114+
public void Dispose ()
115+
{
116+
contentHash.Dispose ();
117+
incrementalHash?.Dispose ();
118+
}
119+
120+
void EnsureScratch (int required)
121+
{
122+
if (scratch.Length < required) {
123+
scratch = new byte [Math.Max (required, scratch.Length * 2)];
124+
}
125+
}
126+
127+
void Write (Sink sink, byte [] data, int offset, int count)
128+
{
129+
if ((sink & Sink.Content) != 0) {
130+
if (count > contentBuffer.Length - contentPosition) {
131+
FlushContent ();
132+
}
133+
if (count > contentBuffer.Length) {
134+
contentHash.AppendData (data, offset, count);
135+
} else {
136+
Buffer.BlockCopy (data, offset, contentBuffer, contentPosition, count);
137+
contentPosition += count;
138+
}
139+
}
140+
if ((sink & Sink.Incremental) != 0 && incrementalHash is not null && incrementalBuffer is not null) {
141+
if (count > incrementalBuffer.Length - incrementalPosition) {
142+
if (incrementalPosition > 0) {
143+
incrementalHash.AppendData (incrementalBuffer, 0, incrementalPosition);
144+
incrementalPosition = 0;
145+
}
146+
}
147+
if (count > incrementalBuffer.Length) {
148+
incrementalHash.AppendData (data, offset, count);
149+
} else {
150+
Buffer.BlockCopy (data, offset, incrementalBuffer, incrementalPosition, count);
151+
incrementalPosition += count;
152+
}
153+
}
154+
}
155+
156+
void FlushContent ()
157+
{
158+
if (contentPosition > 0) {
159+
contentHash.AppendData (contentBuffer, 0, contentPosition);
160+
contentPosition = 0;
161+
}
162+
}
163+
164+
static int Write7BitEncodedInt (byte [] destination, int offset, int value)
165+
{
166+
uint remaining = (uint) value;
167+
while (remaining > 0x7Fu) {
168+
destination [offset++] = (byte) (remaining | ~0x7Fu);
169+
remaining >>= 7;
170+
}
171+
destination [offset++] = (byte) remaining;
172+
return offset;
173+
}
174+
}

0 commit comments

Comments
 (0)