Support proto2 syntax and custom proto file headers

`proto_schema_artifacts` gains two options:

- `syntax: :proto2` emits a proto2 file instead of the default proto3
  (labeling every field `optional` or `repeated`). This is useful when the
  generated messages need to reference proto2 types — `protoc` forbids a
  proto3 message from referencing a proto2 enum.
- `headers:` injects file-level lines (such as `option` declarations)
  verbatim as a contiguous section after the `package` declaration, so
  language-specific options can be set without baking any particular
  convention into the gem.

Author: jwils

elasticgraph-proto_ingestion/README.md

@@ -1,7 +1,9 @@
 # ElasticGraph::ProtoIngestion
 
 An ElasticGraph extension that supports ingesting Protocol Buffer data into ElasticGraph.
-Currently it generates `proto3` Protocol Buffers schema artifacts from ElasticGraph schemas.
+Currently it generates Protocol Buffers schema artifacts from ElasticGraph schemas: it emits
+`proto3` by default and can emit `proto2`, and supports arbitrary file-level headers (such
+as `option` declarations).
 
 ## Dependency Diagram
 
@@ -78,6 +80,54 @@ After running `bundle exec rake schema_artifacts:dump`, ElasticGraph will genera
 
 ## Schema Definition Options
 
+### Protobuf Syntax (`proto2` / `proto3`)
+
+`proto_schema_artifacts` emits `proto3` by default. Pass `syntax: :proto2` to emit a proto2 file
+instead (every field is then labeled `optional` or `repeated`). This is useful when the generated
+messages need to reference proto2 types — for example, `protoc` forbids a `proto3` message from
+referencing a `proto2` enum:
+
+```ruby
+# in config/schema/protobuf.rb
+
+ElasticGraph.define_schema do |schema|
+  schema.proto_schema_artifacts package_name: "myapp.events.v1", syntax: :proto2
+end
+```
+
+### Custom Headers
+
+Pass `headers:` an array of strings to inject file-level lines (such as `option` declarations)
+verbatim, as a contiguous section immediately after the `package` declaration. This lets you set
+language-specific options without the gem baking in any particular convention:
+
+```ruby
+# in config/schema/protobuf.rb
+
+ElasticGraph.define_schema do |schema|
+  schema.proto_schema_artifacts(
+    package_name: "myapp.events.v1",
+    headers: [
+      %(option java_package = "com.myapp.events";),
+      "option java_multiple_files = true;"
+    ]
+  )
+end
+```
+
+produces:
+
+```text
+syntax = "proto3";
+
+package myapp.events.v1;
+
+option java_package = "com.myapp.events";
+option java_multiple_files = true;
+
+// ...messages...
+```
+
 ### Custom Scalar Types
 
 Built-in ElasticGraph scalar types are automatically mapped to proto scalar types.

elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/api_extension.rb

@@ -60,18 +60,42 @@ def self.extended(api)
         # Configures protobuf artifact generation behavior.
         #
         # @param package_name [String] proto package name to emit
+        # @param syntax [Symbol] `:proto3` (default) or `:proto2`
+        # @param headers [Array<String>] file-level header lines (e.g. `option` declarations) rendered
+        #   verbatim after the `package` declaration
         # @return [void]
         #
         # @example Set the proto package name
         #   ElasticGraph.define_schema do |schema|
         #     schema.proto_schema_artifacts package_name: "myapp.events.v1"
         #   end
-        def proto_schema_artifacts(package_name: "elasticgraph")
+        #
+        # @example Emit proto2 with custom file-level options
+        #   ElasticGraph.define_schema do |schema|
+        #     schema.proto_schema_artifacts(
+        #       package_name: "myapp.events.v1",
+        #       syntax: :proto2,
+        #       headers: [
+        #         %(option java_package = "com.myapp.events";),
+        #         "option java_multiple_files = true;"
+        #       ]
+        #     )
+        #   end
+        def proto_schema_artifacts(package_name: "elasticgraph", syntax: :proto3, headers: [])
           if !package_name.is_a?(String) || package_name.empty?
             raise Errors::SchemaError, "`package_name` must be a non-empty String"
           end
+          unless Schema::SUPPORTED_SYNTAXES.include?(syntax.to_s)
+            raise Errors::SchemaError, "`syntax` must be one of #{Schema::SUPPORTED_SYNTAXES.inspect}, got: #{syntax.inspect}"
+          end
+          if !headers.is_a?(Array) || headers.any? { |header| !header.is_a?(String) }
+            raise Errors::SchemaError, "`headers` must be an Array of Strings"
+          end
 
-          proto_ingestion_state.package_name = package_name
+          ingestion_state = proto_ingestion_state
+          ingestion_state.package_name = package_name
+          ingestion_state.syntax = syntax
+          ingestion_state.headers = headers
           nil
         end
 

elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/proto_ingestion_state.rb

@@ -15,11 +15,15 @@ module SchemaDefinition
       class ProtoIngestionState
         # @dynamic package_name, package_name=
         # @dynamic field_number_mappings, field_number_mappings=
-        attr_accessor :package_name, :field_number_mappings
+        # @dynamic syntax, syntax=
+        # @dynamic headers, headers=
+        attr_accessor :package_name, :field_number_mappings, :syntax, :headers
 
         def initialize
           @package_name = "elasticgraph"
           @field_number_mappings = {}
+          @syntax = :proto3
+          @headers = []
         end
       end
     end

elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/results_extension.rb

@@ -47,7 +47,9 @@ def protobuf_schema_generator
             Schema.new(
               state: extension_state,
               package_name: ingestion_state.package_name,
-              proto_field_number_mappings: ingestion_state.field_number_mappings
+              proto_field_number_mappings: ingestion_state.field_number_mappings,
+              syntax: ingestion_state.syntax,
+              headers: ingestion_state.headers
             )
           end
         end

elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema.rb

@@ -15,7 +15,7 @@
 module ElasticGraph
   module ProtoIngestion
     module SchemaDefinition
-      # Builds a `proto3` schema string from an ElasticGraph schema definition.
+      # Builds a `proto2` or `proto3` schema string from an ElasticGraph schema definition.
       class Schema
         # Internal representation of a protobuf field definition.
         #
@@ -61,14 +61,23 @@ class Schema
         #   @return [String]
         FieldNumberMapping = ::Data.define(:field_number, :name_in_index)
 
+        # Protobuf syntaxes this generator can emit.
+        SUPPORTED_SYNTAXES = %w[proto2 proto3].freeze
+
         # @param state [ElasticGraph::SchemaDefinition::State]
         # @param package_name [String]
         # @param proto_field_number_mappings [Hash]
+        # @param syntax [Symbol, String] `:proto3` (default) or `:proto2`; validated by {APIExtension#proto_schema_artifacts}
+        # @param headers [Array<String>] file-level header lines (e.g. `option` declarations) rendered verbatim
         def initialize(
           state:,
           package_name:,
-          proto_field_number_mappings: {}
+          proto_field_number_mappings: {},
+          syntax: :proto3,
+          headers: []
         )
+          @syntax = syntax.to_s
+          @headers = headers
           @state = state
           @package_name = Identifier.package_name(package_name)
           @proto_field_number_mappings_by_message = normalize_proto_field_number_mappings(proto_field_number_mappings)
@@ -84,7 +93,7 @@ def initialize(
           @type_name_by_enum_name = {}
         end
 
-        # Renders the schema as a valid `proto3` file.
+        # Renders the schema as a valid `proto2` or `proto3` file.
         #
         # @return [String]
         def to_proto
@@ -94,8 +103,9 @@ def to_proto
           root_types.each { |type| register_type(type) }
 
           sections = [
-            %(syntax = "proto3";),
+            %(syntax = "#{@syntax}";),
             "package #{@package_name};",
+            *render_headers,
             *render_imports,
             render_definitions
           ]
@@ -418,6 +428,13 @@ def name_taken?(name)
             @enum_definitions_by_name.key?(name)
         end
 
+        # Renders the custom header lines as a single contiguous section (so they are not
+        # blank-line separated). Returns `[]` when no headers were configured.
+        def render_headers
+          return [] if @headers.empty?
+          [@headers.join("\n")]
+        end
+
         # Renders `import` statements for referenced external proto files as a single contiguous
         # section. Returns `[]` when no external types are referenced.
         def render_imports
@@ -464,7 +481,13 @@ def render_message(message_definition)
             lines << "  // No indexed fields were defined for this type."
           else
             message_definition.fields.each do |field|
-              label = field.repeated ? "repeated " : ""
+              # proto2 requires an explicit label on every field; proto3 only uses `repeated`.
+              label =
+                if @syntax == "proto2"
+                  field.repeated ? "repeated " : "optional "
+                else
+                  field.repeated ? "repeated " : ""
+                end
               line = "  #{label}#{field.type} #{field.name} = #{field.field_number};"
               line += " // #{field.comment}" if field.comment
               lines << line

elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/api_extension.rbs

@@ -5,7 +5,7 @@ module ElasticGraph
         PROTO_OPTIONS_BY_BUILT_IN_SCALAR_TYPE: ::Hash[::String, ::Hash[::Symbol, ::String]]
 
         def self.extended: (::ElasticGraph::SchemaDefinition::API & APIExtension) -> void
-        def proto_schema_artifacts: (?package_name: ::String) -> void
+        def proto_schema_artifacts: (?package_name: ::String, ?syntax: ::Symbol, ?headers: ::Array[::String]) -> void
         def configure_proto_field_number_mappings: (untyped) -> void
 
         private

elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/proto_ingestion_state.rbs

@@ -4,6 +4,8 @@ module ElasticGraph
       class ProtoIngestionState
         attr_accessor package_name: ::String
         attr_accessor field_number_mappings: untyped
+        attr_accessor syntax: (::Symbol | ::String)
+        attr_accessor headers: ::Array[::String]
 
         def initialize: () -> void
       end

elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema.rbs

@@ -53,6 +53,10 @@ module ElasticGraph
 
         type fieldNumberMappingsByFieldName = ::Hash[::String, FieldNumberMapping]
 
+        SUPPORTED_SYNTAXES: ::Array[::String]
+
+        @syntax: ::String
+        @headers: ::Array[::String]
         @state: ::ElasticGraph::SchemaDefinition::State
         @package_name: ::String
         @proto_field_number_mappings_by_message: ::Hash[::String, fieldNumberMappingsByFieldName]
@@ -71,7 +75,9 @@ module ElasticGraph
         def initialize: (
           state: ::ElasticGraph::SchemaDefinition::State,
           package_name: ::String,
-          ?proto_field_number_mappings: untyped
+          ?proto_field_number_mappings: untyped,
+          ?syntax: (::Symbol | ::String),
+          ?headers: ::Array[::String]
         ) -> void
 
         def to_proto: () -> ::String
@@ -113,6 +119,7 @@ module ElasticGraph
         ) -> ::String
         def unique_generated_message_name: (::String) -> ::String
         def name_taken?: (::String) -> bool
+        def render_headers: () -> ::Array[::String]
         def render_imports: () -> ::Array[::String]
         def render_definitions: () -> ::String
         def proto_enum_value_name: (::String, ::String) -> ::String

elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/api_extension_spec.rb

@@ -52,6 +52,28 @@ module SchemaDefinition
             end
           }.to raise_error(Errors::SchemaError, a_string_including("`package_name` must be a non-empty String"))
         end
+
+        it "requires `syntax` to be a supported protobuf syntax" do
+          expect {
+            define_proto_schema do |s|
+              s.proto_schema_artifacts syntax: :proto1
+            end
+          }.to raise_error(Errors::SchemaError, a_string_including("`syntax` must be one of"))
+        end
+
+        it "requires `headers` to be an Array of Strings" do
+          expect {
+            define_proto_schema do |s|
+              s.proto_schema_artifacts headers: %(option java_package = "com.example";)
+            end
+          }.to raise_error(Errors::SchemaError, a_string_including("`headers` must be an Array of Strings"))
+
+          expect {
+            define_proto_schema do |s|
+              s.proto_schema_artifacts headers: [:not_a_string]
+            end
+          }.to raise_error(Errors::SchemaError, a_string_including("`headers` must be an Array of Strings"))
+        end
       end
     end
   end

elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/schema_spec.rb

@@ -57,6 +57,71 @@ module SchemaDefinition
           PROTO
         end
 
+        it "emits proto2 syntax with an explicit label on every field when `syntax: :proto2`" do
+          results = define_proto_schema do |s|
+            s.proto_schema_artifacts syntax: :proto2
+
+            s.enum_type "Status" do |t|
+              t.values "ACTIVE", "INACTIVE"
+            end
+
+            s.object_type "Account" do |t|
+              t.field "id", "ID"
+              t.field "status", "Status"
+              t.field "tags", "[String!]!"
+              t.index "accounts"
+            end
+          end
+
+          expect(proto_schema_from(results)).to eq(<<~PROTO)
+            syntax = "proto2";
+
+            package elasticgraph;
+
+            enum Status {
+              STATUS_UNSPECIFIED = 0;
+              STATUS_ACTIVE = 1;
+              STATUS_INACTIVE = 2;
+            }
+
+            message Account {
+              optional string id = 1;
+              optional Status status = 2;
+              repeated string tags = 3;
+            }
+          PROTO
+        end
+
+        it "renders custom `headers` verbatim as a contiguous section after the package declaration" do
+          results = define_proto_schema do |s|
+            s.proto_schema_artifacts(
+              package_name: "myapp.events.v1",
+              headers: [
+                %(option java_package = "com.myapp.events";),
+                "option java_multiple_files = true;"
+              ]
+            )
+
+            s.object_type "Account" do |t|
+              t.field "id", "ID"
+              t.index "accounts"
+            end
+          end
+
+          expect(proto_schema_from(results)).to eq(<<~PROTO)
+            syntax = "proto3";
+
+            package myapp.events.v1;
+
+            option java_package = "com.myapp.events";
+            option java_multiple_files = true;
+
+            message Account {
+              string id = 1;
+            }
+          PROTO
+        end
+
         it "generates a single message for an indexed abstract type, covering its subtypes' fields" do
           results = define_proto_schema do |s|
             s.object_type "Car" do |t|