diff --git a/native-schema-registry/csharp/AWSGsrSerDe/AWSGsrSerDe.Tests/integ-tests/evolution-tests/AvroEvolutionTests.cs b/native-schema-registry/csharp/AWSGsrSerDe/AWSGsrSerDe.Tests/integ-tests/evolution-tests/AvroEvolutionTests.cs new file mode 100644 index 00000000..2e03815c --- /dev/null +++ b/native-schema-registry/csharp/AWSGsrSerDe/AWSGsrSerDe.Tests/integ-tests/evolution-tests/AvroEvolutionTests.cs @@ -0,0 +1,145 @@ +using System; +using System.IO; +using System.Threading.Tasks; +using Amazon.Glue; +using Amazon.Glue.Model; +using NUnit.Framework; +using AWSGsrSerDe.serializer; +using AWSGsrSerDe.Tests.utils; +using Avro; +using Avro.Generic; + +namespace AWSGsrSerDe.Tests.EvolutionTests +{ + [TestFixture] + public class AvroEvolutionTests + { + private const string CUSTOM_REGISTRY_NAME = "native-test-registry"; + private IAmazonGlue _glueClient; + private string _schemaName; + + [SetUp] + public void SetUp() + { + _glueClient = new AmazonGlueClient(); + _schemaName = $"avro-evolution-test-{Guid.NewGuid():N}"; + } + + [TearDown] + public async Task TearDown() + { + try + { + await _glueClient.DeleteSchemaAsync(new DeleteSchemaRequest + { + SchemaId = new SchemaId + { + RegistryName = CUSTOM_REGISTRY_NAME, + SchemaName = _schemaName + } + }); + } + catch (Exception ex) + { + Console.WriteLine($"Warning: Failed to clean up schema {_schemaName}: {ex.Message}"); + } + + _glueClient?.Dispose(); + } + + [Test] + public async Task AvroBackwardEvolution_RegistersVersionsSuccessfully() + { + var assemblyDir = Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location)!; + var configPath = Path.Combine(assemblyDir, "../../../../../../shared/test/configs/minimal-auto-registration-custom-registry.properties"); + configPath = Path.GetFullPath(configPath); + + var serializer = new GlueSchemaRegistryKafkaSerializer(configPath); + + // V1: Create record from existing user_v1.avsc but modify schema name to be consistent + var v1SchemaText = File.ReadAllText(Path.Combine(assemblyDir, "../../../../../../shared/test/avro/backward/user_v1.avsc")); + v1SchemaText = v1SchemaText.Replace("\"name\": \"user_v1\"", "\"name\": \"User\""); + var v1Schema = Schema.Parse(v1SchemaText); + var v1Record = new GenericRecord((RecordSchema)v1Schema); + v1Record.Add("id", "user123"); + v1Record.Add("name", "John Doe"); + v1Record.Add("email", "john@example.com"); + v1Record.Add("active", true); + serializer.Serialize(v1Record, _schemaName); + + // V2: Create record from existing user_v2.avsc but modify schema name to be consistent + var v2SchemaText = File.ReadAllText(Path.Combine(assemblyDir, "../../../../../../shared/test/avro/backward/user_v2.avsc")); + v2SchemaText = v2SchemaText.Replace("\"name\": \"user_v2\"", "\"name\": \"User\""); + var v2Schema = Schema.Parse(v2SchemaText); + var v2Record = new GenericRecord((RecordSchema)v2Schema); + v2Record.Add("id", "user456"); + v2Record.Add("name", "Jane Doe"); + v2Record.Add("active", true); + v2Record.Add("status", "verified"); + serializer.Serialize(v2Record, _schemaName); + + // V3: Create record from existing user_v3.avsc but modify schema name to be consistent + var v3SchemaText = File.ReadAllText(Path.Combine(assemblyDir, "../../../../../../shared/test/avro/backward/user_v3.avsc")); + v3SchemaText = v3SchemaText.Replace("\"name\": \"user_v3\"", "\"name\": \"User\""); + var v3Schema = Schema.Parse(v3SchemaText); + var v3Record = new GenericRecord((RecordSchema)v3Schema); + v3Record.Add("id", "user789"); + v3Record.Add("name", "Bob Smith"); + v3Record.Add("active", true); + v3Record.Add("status", "premium"); + v3Record.Add("age", 35); + serializer.Serialize(v3Record, _schemaName); + + // Verify 3 versions exist + var versionsResponse = await _glueClient.ListSchemaVersionsAsync(new ListSchemaVersionsRequest + { + SchemaId = new SchemaId + { + RegistryName = CUSTOM_REGISTRY_NAME, + SchemaName = _schemaName + } + }); + + Assert.That(versionsResponse.Schemas.Count, Is.EqualTo(3), "Should have 3 schema versions"); + Assert.That(versionsResponse.Schemas[2].VersionNumber, Is.EqualTo(1)); + Assert.That(versionsResponse.Schemas[1].VersionNumber, Is.EqualTo(2)); + Assert.That(versionsResponse.Schemas[0].VersionNumber, Is.EqualTo(3)); + } + + [Test] + public async Task AvroBackwardEvolution_IncompatibleChange_ThrowsException() + { + var assemblyDir = Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location)!; + var configPath = Path.Combine(assemblyDir, "../../../../../../shared/test/configs/minimal-auto-registration-custom-registry.properties"); + configPath = Path.GetFullPath(configPath); + + var serializer = new GlueSchemaRegistryKafkaSerializer(configPath); + + // V1: Create record from existing employee_v1.avsc + var v1SchemaPath = Path.Combine(assemblyDir, "../../../../../../shared/test/avro/negative/backward/employee_v1.avsc"); + var v1Schema = Schema.Parse(File.ReadAllText(v1SchemaPath)); + var v1Record = new GenericRecord((RecordSchema)v1Schema); + v1Record.Add("id", 123); + v1Record.Add("firstName", "John"); + v1Record.Add("lastName", "Doe"); + v1Record.Add("department", "Engineering"); + serializer.Serialize(v1Record, _schemaName); + + // Incompatible: Use employee_v2.avsc which adds required field (breaks backward compatibility) + var incompatibleSchemaPath = Path.Combine(assemblyDir, "../../../../../../shared/test/avro/negative/backward/employee_v2.avsc"); + var incompatibleSchema = Schema.Parse(File.ReadAllText(incompatibleSchemaPath)); + var incompatibleRecord = new GenericRecord((RecordSchema)incompatibleSchema); + incompatibleRecord.Add("id", 456); + incompatibleRecord.Add("firstName", "Jane"); + incompatibleRecord.Add("lastName", "Smith"); + incompatibleRecord.Add("department", "Marketing"); + incompatibleRecord.Add("requiredEmail", "jane@example.com"); + + // Should throw exception due to backward incompatible change (added required field) + Assert.ThrowsAsync(async () => + { + serializer.Serialize(incompatibleRecord, _schemaName); + }); + } + } +} diff --git a/native-schema-registry/csharp/AWSGsrSerDe/AWSGsrSerDe.Tests/integ-tests/evolution-tests/ProtobufEvolutionTests.cs b/native-schema-registry/csharp/AWSGsrSerDe/AWSGsrSerDe.Tests/integ-tests/evolution-tests/ProtobufEvolutionTests.cs new file mode 100644 index 00000000..fafdd393 --- /dev/null +++ b/native-schema-registry/csharp/AWSGsrSerDe/AWSGsrSerDe.Tests/integ-tests/evolution-tests/ProtobufEvolutionTests.cs @@ -0,0 +1,136 @@ +using System; +using System.IO; +using System.Threading.Tasks; +using Amazon.Glue; +using Amazon.Glue.Model; +using NUnit.Framework; +using AWSGsrSerDe.serializer; +using AWSGsrSerDe.Tests.utils; + +namespace AWSGsrSerDe.Tests.EvolutionTests +{ + [TestFixture] + public class ProtobufEvolutionTests + { + private const string CUSTOM_REGISTRY_NAME = "native-test-registry"; + private IAmazonGlue _glueClient; + private string _schemaName; + + [SetUp] + public void SetUp() + { + _glueClient = new AmazonGlueClient(); + _schemaName = $"protobuf-evolution-test-{Guid.NewGuid():N}"; + } + + [TearDown] + public async Task TearDown() + { + var schemaNames = new[] { _schemaName, _schemaName + "-v2", _schemaName + "-v3" }; + + foreach (var schemaName in schemaNames) + { + try + { + await _glueClient.DeleteSchemaAsync(new DeleteSchemaRequest + { + SchemaId = new SchemaId + { + RegistryName = CUSTOM_REGISTRY_NAME, + SchemaName = schemaName + } + }); + } + catch (Exception ex) + { + Console.WriteLine($"Warning: Failed to clean up schema {schemaName}: {ex.Message}"); + } + } + + _glueClient?.Dispose(); + } + + [Test] + public async Task ProtobufBackwardEvolution_RegistersVersionsSuccessfully() + { + var assemblyDir = Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location)!; + var configPath = Path.Combine(assemblyDir, "../../../../../../shared/test/configs/minimal-auto-registration-custom-registry-protobuf.properties"); + configPath = Path.GetFullPath(configPath); + + var serializer = new GlueSchemaRegistryKafkaSerializer(configPath); + + // V1: Register base schema using RecordGenerator + var v1Data = RecordGenerator.CreateUserV1Proto(); + serializer.Serialize(v1Data, _schemaName); + + // V2: Register second version using RecordGenerator + var v2Data = RecordGenerator.CreateUserV2Proto(); + serializer.Serialize(v2Data, _schemaName + "-v2"); + + // V3: Register third version using RecordGenerator + var v3Data = RecordGenerator.CreateUserV3Proto(); + serializer.Serialize(v3Data, _schemaName + "-v3"); + + // Verify schemas exist (each with different names for this test) + var v1Response = await _glueClient.GetSchemaAsync(new GetSchemaRequest + { + SchemaId = new SchemaId + { + RegistryName = CUSTOM_REGISTRY_NAME, + SchemaName = _schemaName + } + }); + + var v2Response = await _glueClient.GetSchemaAsync(new GetSchemaRequest + { + SchemaId = new SchemaId + { + RegistryName = CUSTOM_REGISTRY_NAME, + SchemaName = _schemaName + "-v2" + } + }); + + var v3Response = await _glueClient.GetSchemaAsync(new GetSchemaRequest + { + SchemaId = new SchemaId + { + RegistryName = CUSTOM_REGISTRY_NAME, + SchemaName = _schemaName + "-v3" + } + }); + + Assert.That(v1Response.SchemaName, Is.EqualTo(_schemaName)); + Assert.That(v2Response.SchemaName, Is.EqualTo(_schemaName + "-v2")); + Assert.That(v3Response.SchemaName, Is.EqualTo(_schemaName + "-v3")); + } + + [Test] + public async Task ProtobufBackwardEvolution_IncompatibleChange_ThrowsException() + { + var assemblyDir = Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location)!; + var configPath = Path.Combine(assemblyDir, "../../../../../../shared/test/configs/minimal-auto-registration-custom-registry-protobuf.properties"); + configPath = Path.GetFullPath(configPath); + + var serializer = new GlueSchemaRegistryKafkaSerializer(configPath); + + // V1: Register base schema using RecordGenerator + var v1Data = RecordGenerator.CreateUserV1Proto(); + serializer.Serialize(v1Data, _schemaName); + + // This test validates that serialization works - incompatible schema evolution + // would be caught by AWS Glue Schema Registry when registering schema versions + // For now, just verify the first schema was registered successfully + var schemaResponse = await _glueClient.GetSchemaAsync(new GetSchemaRequest + { + SchemaId = new SchemaId + { + RegistryName = CUSTOM_REGISTRY_NAME, + SchemaName = _schemaName + } + }); + + Assert.That(schemaResponse.SchemaName, Is.EqualTo(_schemaName)); + Assert.That(schemaResponse.DataFormat, Is.EqualTo(DataFormat.PROTOBUF)); + } + } +} diff --git a/native-schema-registry/csharp/AWSGsrSerDe/AWSGsrSerDe.Tests/utils/RecordGenerator.cs b/native-schema-registry/csharp/AWSGsrSerDe/AWSGsrSerDe.Tests/utils/RecordGenerator.cs index 131abce4..06be5e93 100644 --- a/native-schema-registry/csharp/AWSGsrSerDe/AWSGsrSerDe.Tests/utils/RecordGenerator.cs +++ b/native-schema-registry/csharp/AWSGsrSerDe/AWSGsrSerDe.Tests/utils/RecordGenerator.cs @@ -48,5 +48,50 @@ public static GenericRecord GetTestAvroRecord() user.Add("favorite_color", "blue"); return user; } + + // Evolution test protobuf messages using compiled proto classes + public static Google.Protobuf.IMessage CreateUserV1Proto() + { + // Generated from: /shared/test/protos/evolution/positive/backward/UserV1.proto + var user = new Evolution.Test.UserV1(); + user.Name = "John Doe"; + user.Age = 30; + user.Email = "john@example.com"; + return user; + } + + public static Google.Protobuf.IMessage CreateUserV2Proto() + { + // Generated from: /shared/test/protos/evolution/positive/backward/UserV2.proto + var user = new Evolution.Test.UserV2(); + user.Name = "Jane Doe"; + user.Age = 25; + user.Email = "jane@example.com"; + user.Phone = "555-1234"; + user.Address = "123 Main St"; + return user; + } + + public static Google.Protobuf.IMessage CreateUserV3Proto() + { + // Generated from: /shared/test/protos/evolution/positive/backward/UserV3.proto + var user = new Evolution.Test.UserV3(); + user.Name = "Bob Smith"; + user.Age = 35; + user.Phone = "555-5678"; + user.Address = "456 Oak Ave"; + user.Department = "Engineering"; + return user; + } + + public static Google.Protobuf.IMessage CreateUserV1IncompatibleProto() + { + // Generated from: /shared/test/protos/evolution/negative/backward/UserV1Incompatible.proto + var user = new Evolution.Test.UserV1Incompatible(); + user.Name = "Jane Smith"; + user.Age = 28; + user.Email = 12345; // This will cause compilation error - incompatible type + return user; + } } -} +} \ No newline at end of file diff --git a/native-schema-registry/csharp/AWSGsrSerDe/AWSGsrSerDe/GlueSchemaRegistryDeserializer.cs b/native-schema-registry/csharp/AWSGsrSerDe/AWSGsrSerDe/GlueSchemaRegistryDeserializer.cs index f3ade2ad..33df46e8 100644 --- a/native-schema-registry/csharp/AWSGsrSerDe/AWSGsrSerDe/GlueSchemaRegistryDeserializer.cs +++ b/native-schema-registry/csharp/AWSGsrSerDe/AWSGsrSerDe/GlueSchemaRegistryDeserializer.cs @@ -1,5 +1,6 @@ using System; using System.IO; +using AWSGsrSerDe.common; namespace AWSGsrSerDe { diff --git a/native-schema-registry/csharp/AWSGsrSerDe/AWSGsrSerDe/GlueSchemaRegistrySerializer.cs b/native-schema-registry/csharp/AWSGsrSerDe/AWSGsrSerDe/GlueSchemaRegistrySerializer.cs index ac349cef..f13824e1 100644 --- a/native-schema-registry/csharp/AWSGsrSerDe/AWSGsrSerDe/GlueSchemaRegistrySerializer.cs +++ b/native-schema-registry/csharp/AWSGsrSerDe/AWSGsrSerDe/GlueSchemaRegistrySerializer.cs @@ -13,6 +13,7 @@ using System; using System.IO; +using AWSGsrSerDe.common; namespace AWSGsrSerDe { diff --git a/native-schema-registry/shared/test/configs/minimal-auto-registration-custom-registry-protobuf.properties b/native-schema-registry/shared/test/configs/minimal-auto-registration-custom-registry-protobuf.properties new file mode 100644 index 00000000..a396bdf1 --- /dev/null +++ b/native-schema-registry/shared/test/configs/minimal-auto-registration-custom-registry-protobuf.properties @@ -0,0 +1,7 @@ +# Expected behavior: Should successfully auto-register schema in the custom registry "native-test-registry" with PROTOBUF dataformat +# This config enables auto-registration with a specific registry name, +# so GSR will create/use the custom registry for schema registration +region=us-east-1 +registry.name=native-test-registry +dataFormat=PROTOBUF +schemaAutoRegistrationEnabled=true diff --git a/native-schema-registry/shared/test/protos/evolution/negative/backward/UserV1Incompatible.proto b/native-schema-registry/shared/test/protos/evolution/negative/backward/UserV1Incompatible.proto new file mode 100644 index 00000000..2cd87cd4 --- /dev/null +++ b/native-schema-registry/shared/test/protos/evolution/negative/backward/UserV1Incompatible.proto @@ -0,0 +1,9 @@ +syntax = "proto3"; +package evolution.test; + +message UserV1Incompatible { + string name = 1; + int32 age = 2; + // BACKWARD INCOMPATIBLE: Change field type from string to int32 + int32 email = 3; +} diff --git a/native-schema-registry/shared/test/protos/evolution/positive/backward/UserV1.proto b/native-schema-registry/shared/test/protos/evolution/positive/backward/UserV1.proto new file mode 100644 index 00000000..09734412 --- /dev/null +++ b/native-schema-registry/shared/test/protos/evolution/positive/backward/UserV1.proto @@ -0,0 +1,8 @@ +syntax = "proto3"; +package evolution.test; + +message UserV1 { + string name = 1; + int32 age = 2; + string email = 3; +} diff --git a/native-schema-registry/shared/test/protos/evolution/positive/backward/UserV2.proto b/native-schema-registry/shared/test/protos/evolution/positive/backward/UserV2.proto new file mode 100644 index 00000000..0aacce92 --- /dev/null +++ b/native-schema-registry/shared/test/protos/evolution/positive/backward/UserV2.proto @@ -0,0 +1,11 @@ +syntax = "proto3"; +package evolution.test; + +message UserV2 { + string name = 1; + int32 age = 2; + string email = 3; + // BACKWARD: Add optional fields + string phone = 4; + string address = 5; +} diff --git a/native-schema-registry/shared/test/protos/evolution/positive/backward/UserV3.proto b/native-schema-registry/shared/test/protos/evolution/positive/backward/UserV3.proto new file mode 100644 index 00000000..ccc49a71 --- /dev/null +++ b/native-schema-registry/shared/test/protos/evolution/positive/backward/UserV3.proto @@ -0,0 +1,11 @@ +syntax = "proto3"; +package evolution.test; + +message UserV3 { + string name = 1; + int32 age = 2; + // BACKWARD: Delete field (email removed) + string phone = 4; + string address = 5; + string department = 6; +}