What version of protobuf and what language are you using?
3.36.0
Version: main/v33.0/etc (NOTE: please update to the
latest supported version of
protoc/runtime possible beforehand to attempt to resolve your problem)
Language: C#
What
supported operating system version
are you using (e.g. Linux, Windows) ?
This happened in Linux and was reproduced in a mac machine
What
supported runtime / compiler version
are you using (e.g. python version, gcc version)
.Net8
What did you do?
Steps to reproduce the behavior:
Demo application
public class ProtobufExtensionCacheRaceReproTests
{
private const int MaxAttempts = 15;
private static readonly TimeSpan PerAttemptTimeout = TimeSpan.FromSeconds(20);
private readonly ITestOutputHelper _output;
public ProtobufExtensionCacheRaceReproTests(ITestOutputHelper output)
{
_output = output;
}
[Fact]
[Trait("Category", "ProtobufRaceRepro")]
public async Task ConcurrentFirstTouchOfSharedProtoDependencies_CorruptsExtensionCache()
{
var reproDllPath = ResolveReproDllPath();
File.Exists(reproDllPath).Should().BeTrue(
$"the repro harness must be built alongside the test project at '{reproDllPath}'");
for (var attempt = 1; attempt <= MaxAttempts; attempt++)
{
var (exitCode, stdout) = await RunReproProcessAsync(reproDllPath);
_output.WriteLine($"Attempt {attempt}/{MaxAttempts}: exit={exitCode}");
_output.WriteLine(stdout);
if (exitCode == 1 && stdout.Contains("RACE_REPRODUCED"))
{
return;
}
}
Assert.Fail(
$"Did not reproduce the Google.Protobuf extension-cache race in {MaxAttempts} attempts. " +
"This is a timing-dependent upstream bug (protobuf 3.36.0 FileDescriptor.GetAllDependedExtensions " +
"writes to a shared static Dictionary without a lock) - if this consistently fails to reproduce, " +
"either the dependency has been downgraded/patched, or the machine is too fast/slow for the current " +
"thread count to collide reliably.");
}
private static string ResolveReproDllPath()
{
var baseDir = AppContext.BaseDirectory;
var reproDir = baseDir.Replace(
$"{Path.DirectorySeparatorChar}TTD.Infra.Janus.UT{Path.DirectorySeparatorChar}",
$"{Path.DirectorySeparatorChar}TTD.Infra.Janus.ProtobufRaceRepro{Path.DirectorySeparatorChar}");
return Path.Combine(reproDir, "TTD.Infra.Janus.ProtobufRaceRepro.dll");
}
private static async Task<(int ExitCode, string Stdout)> RunReproProcessAsync(string reproDllPath)
{
using var process = new Process
{
StartInfo = new ProcessStartInfo
{
FileName = "dotnet",
Arguments = $"exec \"{reproDllPath}\"",
RedirectStandardOutput = true,
RedirectStandardError = true,
UseShellExecute = false,
},
};
process.Start();
var stdoutTask = process.StandardOutput.ReadToEndAsync();
var stderrTask = process.StandardError.ReadToEndAsync();
using var cts = new CancellationTokenSource(PerAttemptTimeout);
await process.WaitForExitAsync(cts.Token);
var stdout = await stdoutTask;
var stderr = await stderrTask;
return (process.ExitCode, stdout + stderr);
}
}
UT to run the application and check for repro
public class ProtobufExtensionCacheRaceReproTests
{
private const int MaxAttempts = 15;
private static readonly TimeSpan PerAttemptTimeout = TimeSpan.FromSeconds(20);
private readonly ITestOutputHelper _output;
public ProtobufExtensionCacheRaceReproTests(ITestOutputHelper output)
{
_output = output;
}
[Fact]
[Trait("Category", "ProtobufRaceRepro")]
public async Task ConcurrentFirstTouchOfSharedProtoDependencies_CorruptsExtensionCache()
{
var reproDllPath = ResolveReproDllPath();
File.Exists(reproDllPath).Should().BeTrue(
$"the repro harness must be built alongside the test project at '{reproDllPath}'");
for (var attempt = 1; attempt <= MaxAttempts; attempt++)
{
var (exitCode, stdout) = await RunReproProcessAsync(reproDllPath);
_output.WriteLine($"Attempt {attempt}/{MaxAttempts}: exit={exitCode}");
_output.WriteLine(stdout);
if (exitCode == 1 && stdout.Contains("RACE_REPRODUCED"))
{
return;
}
}
Assert.Fail(
$"Did not reproduce the Google.Protobuf extension-cache race in {MaxAttempts} attempts. " +
"This is a timing-dependent upstream bug (protobuf 3.36.0 FileDescriptor.GetAllDependedExtensions " +
"writes to a shared static Dictionary without a lock) - if this consistently fails to reproduce, " +
"either the dependency has been downgraded/patched, or the machine is too fast/slow for the current " +
"thread count to collide reliably.");
}
private static string ResolveReproDllPath()
{
var baseDir = AppContext.BaseDirectory;
var reproDir = baseDir.Replace(
$"{Path.DirectorySeparatorChar}TTD.Infra.Janus.UT{Path.DirectorySeparatorChar}",
$"{Path.DirectorySeparatorChar}TTD.Infra.Janus.ProtobufRaceRepro{Path.DirectorySeparatorChar}");
return Path.Combine(reproDir, "TTD.Infra.Janus.ProtobufRaceRepro.dll");
}
private static async Task<(int ExitCode, string Stdout)> RunReproProcessAsync(string reproDllPath)
{
using var process = new Process
{
StartInfo = new ProcessStartInfo
{
FileName = "dotnet",
Arguments = $"exec \"{reproDllPath}\"",
RedirectStandardOutput = true,
RedirectStandardError = true,
UseShellExecute = false,
},
};
process.Start();
var stdoutTask = process.StandardOutput.ReadToEndAsync();
var stderrTask = process.StandardError.ReadToEndAsync();
using var cts = new CancellationTokenSource(PerAttemptTimeout);
await process.WaitForExitAsync(cts.Token);
var stdout = await stdoutTask;
var stderr = await stderrTask;
return (process.ExitCode, stdout + stderr);
}
}
Output
Attempt 1/15: exit=1
RACE_REPRODUCED
--- System.IndexOutOfRangeException: Index was outside the bounds of the array.
at System.Collections.Generic.Dictionary`2.TryInsert(TKey key, TValue value, InsertionBehavior behavior)
at System.Collections.Generic.Dictionary`2.set_Item(TKey key, TValue value)
at Google.Protobuf.Reflection.FileDescriptor.GetAllDependedExtensions(FileDescriptor descriptor)
at System.Linq.Enumerable.SelectManySingleSelectorIterator`2.MoveNext()
at System.Linq.Enumerable.DistinctIterator`1.MoveNext()
at System.Collections.Generic.List`1.AddRange(IEnumerable`1 collection)
at System.Linq.Enumerable.ConcatIterator`1.ToList()
at Google.Protobuf.Reflection.FileDescriptor.GetAllExtensions(FileDescriptor[] dependencies, GeneratedClrTypeInfo generatedInfo)
at Google.Protobuf.Reflection.FileDescriptor.FromGeneratedCode(Byte[] descriptorData, FileDescriptor[] dependencies, GeneratedClrTypeInfo generatedCodeInfo)
at Program.<>c__DisplayClass0_0.<<Main>$>g__BuildFile|0(String name, FileDescriptor[] dependencies) in /Users/menna.elmesalmy/dev/workspace/TTD.Infra.Janus/src/TTD.Infra.Janus.ProtobufRaceRepro/Program.cs:line 58
at Program.<>c__DisplayClass0_0.<<Main>$>b__4() in /Users/menna.elmesalmy/dev/workspace/TTD.Infra.Janus/src/TTD.Infra.Janus.ProtobufRaceRepro/Program.cs:line 83
What did you expect to see
No exceptions in the application.
What did you see instead?
Our redis polling keeps failing
[08/09/2026 11:54:38 ERR 0003] <RedisStreamTelemetry.StreamReadError> Error reading updates from Redis stream stream-1
ERROR:The type initializer for 'ProtoConfigReflection' threw an exception.
System.TypeInitializationException: The type initializer for 'ProtoConfigReflection' threw an exception.
---> System.InvalidOperationException: Operations that change non-concurrent collections must have exclusive access. A concurrent update was performed on this collection and corrupted its state. The collection's state is no longer correct.
at System.Collections.Generic.Dictionary`2.FindValue(TKey key)
at System.Collections.Generic.Dictionary`2.TryGetValue(TKey key, TValue& value)
at Google.Protobuf.Reflection.FileDescriptor.GetAllDependedExtensions(FileDescriptor descriptor)
at System.Linq.Enumerable.SelectManySingleSelectorIterator`2.MoveNext()
at System.Collections.Generic.List`1.AddRange(IEnumerable`1 collection)
at System.Linq.Enumerable.ConcatIterator`1.ToList()
at Google.Protobuf.Reflection.FileDescriptor.GetAllDependedExtensions(FileDescriptor descriptor)
at System.Linq.Enumerable.SelectManySingleSelectorIterator`2.MoveNext()
at System.Linq.Enumerable.DistinctIterator`1.MoveNext()
at System.Collections.Generic.List`1.AddRange(IEnumerable`1 collection)
at System.Linq.Enumerable.ConcatIterator`1.ToList()
at Google.Protobuf.Reflection.FileDescriptor.GetAllExtensions(FileDescriptor[] dependencies, GeneratedClrTypeInfo generatedInfo)
at Google.Protobuf.Reflection.FileDescriptor.FromGeneratedCode(Byte[] descriptorData, FileDescriptor[] dependencies, GeneratedClrTypeInfo generatedCodeInfo)
at ProtoConfigReflection..cctor() in /ProtoConfigReflection/obj/Release/net8.0/Protos/ConfigUpdatesStream.cs:line 45
--- End of inner exception stack trace ---
at ProtoConfigMessage.get_Descriptor() in Contract/obj/Release/net8.0/Protos/ConfigUpdatesStream.cs:line 98
at ConfigStreamMessage.ToString() in /Contract/obj/Release/net8.0/Protos/ConfigUpdatesStream.cs:line ...
Anything else we should know about your project / environment
N/A
This seems to be solvable with a lock or ConcurrentDictionary, any concern with me raising a PR to do that?
Changing
private static readonly Dictionary<string, List<Extension>> allDependedExtensionsCache = new();
to
private static readonly ConcurrentDictionary<string, List<Extension>> allDependedExtensionsCache = new();
does the trick in my intial testing
What version of protobuf and what language are you using?
3.36.0
Version: main/v33.0/etc (NOTE: please update to the
latest supported version of
protoc/runtime possible beforehand to attempt to resolve your problem)
Language: C#
What
supported operating system version
are you using (e.g. Linux, Windows) ?
This happened in Linux and was reproduced in a mac machine
What
supported runtime / compiler version
are you using (e.g. python version, gcc version)
.Net8
What did you do?
Steps to reproduce the behavior:
Demo application
UT to run the application and check for repro
Output
What did you expect to see
No exceptions in the application.
What did you see instead?
Our redis polling keeps failing
Anything else we should know about your project / environment
N/A
This seems to be solvable with a lock or ConcurrentDictionary, any concern with me raising a PR to do that?
Changing
to
does the trick in my intial testing