From fce5338d55b47ebe208bde723a4c6d28ac340851 Mon Sep 17 00:00:00 2001 From: Ahmed Abualsaud Date: Tue, 13 May 2025 11:54:08 -0400 Subject: [PATCH 1/2] add tests --- .../protobuf/ProtoMessageSchemaTest.java | 24 ++++++++++ .../extensions/protobuf/TestProtoSchemas.java | 46 +++++++++++++++++++ .../test/proto/proto3_schema_messages.proto | 18 ++++++++ 3 files changed, 88 insertions(+) diff --git a/sdks/java/extensions/protobuf/src/test/java/org/apache/beam/sdk/extensions/protobuf/ProtoMessageSchemaTest.java b/sdks/java/extensions/protobuf/src/test/java/org/apache/beam/sdk/extensions/protobuf/ProtoMessageSchemaTest.java index 3b4568f1fac7..64d3dc7c3fb7 100644 --- a/sdks/java/extensions/protobuf/src/test/java/org/apache/beam/sdk/extensions/protobuf/ProtoMessageSchemaTest.java +++ b/sdks/java/extensions/protobuf/src/test/java/org/apache/beam/sdk/extensions/protobuf/ProtoMessageSchemaTest.java @@ -29,6 +29,9 @@ import static org.apache.beam.sdk.extensions.protobuf.TestProtoSchemas.NULL_MAP_PRIMITIVE_ROW; import static org.apache.beam.sdk.extensions.protobuf.TestProtoSchemas.NULL_REPEATED_PROTO; import static org.apache.beam.sdk.extensions.protobuf.TestProtoSchemas.NULL_REPEATED_ROW; +import static org.apache.beam.sdk.extensions.protobuf.TestProtoSchemas.NULLABLE_PRIMITIVE_PROTO; +import static org.apache.beam.sdk.extensions.protobuf.TestProtoSchemas.NULLABLE_PRIMITIVE_ROW; +import static org.apache.beam.sdk.extensions.protobuf.TestProtoSchemas.NULLABLE_PRIMITIVE_SCHEMA; import static org.apache.beam.sdk.extensions.protobuf.TestProtoSchemas.ONEOF_PROTO_BOOL; import static org.apache.beam.sdk.extensions.protobuf.TestProtoSchemas.ONEOF_PROTO_INT32; import static org.apache.beam.sdk.extensions.protobuf.TestProtoSchemas.ONEOF_PROTO_PRIMITIVE; @@ -75,6 +78,7 @@ import org.apache.beam.sdk.extensions.protobuf.Proto3SchemaMessages.MapPrimitive; import org.apache.beam.sdk.extensions.protobuf.Proto3SchemaMessages.Nested; import org.apache.beam.sdk.extensions.protobuf.Proto3SchemaMessages.NonContiguousOneOf; +import org.apache.beam.sdk.extensions.protobuf.Proto3SchemaMessages.NullablePrimitive; import org.apache.beam.sdk.extensions.protobuf.Proto3SchemaMessages.OneOf; import org.apache.beam.sdk.extensions.protobuf.Proto3SchemaMessages.OuterOneOf; import org.apache.beam.sdk.extensions.protobuf.Proto3SchemaMessages.Primitive; @@ -116,6 +120,26 @@ public void testPrimitiveRowToProto() { assertEquals(PRIMITIVE_PROTO, fromRow.apply(PRIMITIVE_ROW)); } + @Test + public void testNullablePrimitiveSchema() { + Schema schema = new ProtoMessageSchema().schemaFor(TypeDescriptor.of(NullablePrimitive.class)); + assertEquals(NULLABLE_PRIMITIVE_SCHEMA, schema); + } + + @Test + public void testNullablePrimitiveProtoToRow() { + SerializableFunction toRow = + new ProtoMessageSchema().toRowFunction(TypeDescriptor.of(NullablePrimitive.class)); + assertEquals(NULLABLE_PRIMITIVE_ROW, toRow.apply(NULLABLE_PRIMITIVE_PROTO)); + } + + @Test + public void testNullablePrimitiveRowToProto() { + SerializableFunction fromRow = + new ProtoMessageSchema().fromRowFunction(TypeDescriptor.of(NullablePrimitive.class)); + assertEquals(NULLABLE_PRIMITIVE_PROTO, fromRow.apply(NULLABLE_PRIMITIVE_ROW)); + } + @Test public void testOptionalPrimitiveSchema() { Schema schema = new ProtoMessageSchema().schemaFor(TypeDescriptor.of(OptionalPrimitive.class)); diff --git a/sdks/java/extensions/protobuf/src/test/java/org/apache/beam/sdk/extensions/protobuf/TestProtoSchemas.java b/sdks/java/extensions/protobuf/src/test/java/org/apache/beam/sdk/extensions/protobuf/TestProtoSchemas.java index 234ae8cd6852..0a68200b75d4 100644 --- a/sdks/java/extensions/protobuf/src/test/java/org/apache/beam/sdk/extensions/protobuf/TestProtoSchemas.java +++ b/sdks/java/extensions/protobuf/src/test/java/org/apache/beam/sdk/extensions/protobuf/TestProtoSchemas.java @@ -44,6 +44,7 @@ import org.apache.beam.sdk.extensions.protobuf.Proto3SchemaMessages.MapPrimitive; import org.apache.beam.sdk.extensions.protobuf.Proto3SchemaMessages.Nested; import org.apache.beam.sdk.extensions.protobuf.Proto3SchemaMessages.NonContiguousOneOf; +import org.apache.beam.sdk.extensions.protobuf.Proto3SchemaMessages.NullablePrimitive; import org.apache.beam.sdk.extensions.protobuf.Proto3SchemaMessages.OneOf; import org.apache.beam.sdk.extensions.protobuf.Proto3SchemaMessages.OuterOneOf; import org.apache.beam.sdk.extensions.protobuf.Proto3SchemaMessages.Primitive; @@ -111,6 +112,47 @@ static Schema.Options withTypeName(String typeName) { "proto3_schema_messages.Primitive")) .build(); + static final Schema NULLABLE_PRIMITIVE_SCHEMA = + Schema.builder() + .addField(withFieldNumber("primitive_double", FieldType.DOUBLE, 1).withNullable(true)) + .addField(withFieldNumber("primitive_float", FieldType.FLOAT, 2).withNullable(true)) + .addField(withFieldNumber("primitive_int32", FieldType.INT32, 3).withNullable(true)) + .addField(withFieldNumber("primitive_int64", FieldType.INT64, 4).withNullable(true)) + .addField( + withFieldNumber("primitive_uint32", FieldType.logicalType(new UInt32()), 5) + .withNullable(true)) + .addField( + withFieldNumber("primitive_uint64", FieldType.logicalType(new UInt64()), 6) + .withNullable(true)) + .addField( + withFieldNumber("primitive_sint32", FieldType.logicalType(new SInt32()), 7) + .withNullable(true)) + .addField( + withFieldNumber("primitive_sint64", FieldType.logicalType(new SInt64()), 8) + .withNullable(true)) + .addField( + withFieldNumber("primitive_fixed32", FieldType.logicalType(new Fixed32()), 9) + .withNullable(true)) + .addField( + withFieldNumber("primitive_fixed64", FieldType.logicalType(new Fixed64()), 10) + .withNullable(true)) + .addField( + withFieldNumber("primitive_sfixed32", FieldType.logicalType(new SFixed32()), 11) + .withNullable(true)) + .addField( + withFieldNumber("primitive_sfixed64", FieldType.logicalType(new SFixed64()), 12) + .withNullable(true)) + .addField(withFieldNumber("primitive_bool", FieldType.BOOLEAN, 13).withNullable(true)) + .addField(withFieldNumber("primitive_string", FieldType.STRING, 14).withNullable(true)) + .addField(withFieldNumber("primitive_bytes", FieldType.BYTES, 15).withNullable(true)) + .setOptions( + Schema.Options.builder() + .setOption( + SCHEMA_OPTION_META_TYPE_NAME, + FieldType.STRING, + "proto3_schema_messages.NullablePrimitive")) + .build(); + static final Schema OPTIONAL_PRIMITIVE_SCHEMA = Schema.builder() .addField(withFieldNumber("primitive_int32", FieldType.INT32, 1)) @@ -181,6 +223,10 @@ static Schema.Options withTypeName(String typeName) { .setPrimitiveBytes(ByteString.copyFrom(BYTE_ARRAY)) .build(); + static final NullablePrimitive NULLABLE_PRIMITIVE_PROTO = NullablePrimitive.newBuilder().build(); + + static final Row NULLABLE_PRIMITIVE_ROW = Row.nullRow(NULLABLE_PRIMITIVE_SCHEMA); + // A sample instance of the row. static final Row OPTIONAL_PRIMITIVE_ROW = Row.withSchema(OPTIONAL_PRIMITIVE_SCHEMA).addValues(32, true, "horsey", BYTE_ARRAY).build(); diff --git a/sdks/java/extensions/protobuf/src/test/proto/proto3_schema_messages.proto b/sdks/java/extensions/protobuf/src/test/proto/proto3_schema_messages.proto index 6c8627c130f6..02209631f190 100644 --- a/sdks/java/extensions/protobuf/src/test/proto/proto3_schema_messages.proto +++ b/sdks/java/extensions/protobuf/src/test/proto/proto3_schema_messages.proto @@ -51,6 +51,24 @@ message Primitive { bytes primitive_bytes = 15; } +message NullablePrimitive { + optional double primitive_double = 1; + optional float primitive_float = 2; + optional int32 primitive_int32 = 3; + optional int64 primitive_int64 = 4; + optional uint32 primitive_uint32 = 5; + optional uint64 primitive_uint64 = 6; + optional sint32 primitive_sint32 = 7; + optional sint64 primitive_sint64 = 8; + optional fixed32 primitive_fixed32 = 9; + optional fixed64 primitive_fixed64 = 10; + optional sfixed32 primitive_sfixed32 = 11; + optional sfixed64 primitive_sfixed64 = 12; + optional bool primitive_bool = 13; + optional string primitive_string = 14; + optional bytes primitive_bytes = 15; +} + message RepeatPrimitive { repeated double repeated_double = 1; repeated float repeated_float = 2; From 2527e1d2eca0c0e97cb53368e904f7b050c0bb5d Mon Sep 17 00:00:00 2001 From: Ahmed Abualsaud Date: Tue, 13 May 2025 15:59:59 -0400 Subject: [PATCH 2/2] spotless --- .../sdk/extensions/protobuf/ProtoMessageSchemaTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sdks/java/extensions/protobuf/src/test/java/org/apache/beam/sdk/extensions/protobuf/ProtoMessageSchemaTest.java b/sdks/java/extensions/protobuf/src/test/java/org/apache/beam/sdk/extensions/protobuf/ProtoMessageSchemaTest.java index 64d3dc7c3fb7..f948c9aa0b47 100644 --- a/sdks/java/extensions/protobuf/src/test/java/org/apache/beam/sdk/extensions/protobuf/ProtoMessageSchemaTest.java +++ b/sdks/java/extensions/protobuf/src/test/java/org/apache/beam/sdk/extensions/protobuf/ProtoMessageSchemaTest.java @@ -25,13 +25,13 @@ import static org.apache.beam.sdk.extensions.protobuf.TestProtoSchemas.NESTED_SCHEMA; import static org.apache.beam.sdk.extensions.protobuf.TestProtoSchemas.NONCONTIGUOUS_ONEOF_PROTO; import static org.apache.beam.sdk.extensions.protobuf.TestProtoSchemas.NONCONTIGUOUS_ONEOF_ROW; +import static org.apache.beam.sdk.extensions.protobuf.TestProtoSchemas.NULLABLE_PRIMITIVE_PROTO; +import static org.apache.beam.sdk.extensions.protobuf.TestProtoSchemas.NULLABLE_PRIMITIVE_ROW; +import static org.apache.beam.sdk.extensions.protobuf.TestProtoSchemas.NULLABLE_PRIMITIVE_SCHEMA; import static org.apache.beam.sdk.extensions.protobuf.TestProtoSchemas.NULL_MAP_PRIMITIVE_PROTO; import static org.apache.beam.sdk.extensions.protobuf.TestProtoSchemas.NULL_MAP_PRIMITIVE_ROW; import static org.apache.beam.sdk.extensions.protobuf.TestProtoSchemas.NULL_REPEATED_PROTO; import static org.apache.beam.sdk.extensions.protobuf.TestProtoSchemas.NULL_REPEATED_ROW; -import static org.apache.beam.sdk.extensions.protobuf.TestProtoSchemas.NULLABLE_PRIMITIVE_PROTO; -import static org.apache.beam.sdk.extensions.protobuf.TestProtoSchemas.NULLABLE_PRIMITIVE_ROW; -import static org.apache.beam.sdk.extensions.protobuf.TestProtoSchemas.NULLABLE_PRIMITIVE_SCHEMA; import static org.apache.beam.sdk.extensions.protobuf.TestProtoSchemas.ONEOF_PROTO_BOOL; import static org.apache.beam.sdk.extensions.protobuf.TestProtoSchemas.ONEOF_PROTO_INT32; import static org.apache.beam.sdk.extensions.protobuf.TestProtoSchemas.ONEOF_PROTO_PRIMITIVE;