Reference external proto enum types

Some schemas already have canonical protobuf enum definitions, and
generating duplicate enums in `schema.proto` makes consumers deal with
parallel types. Passing `proto:` and `import:` to `external_proto_enum`
imports the named proto file and references the existing enum type
directly instead of generating a local definition.

A referenced enum must have exactly one option-free source whose values
match the ElasticGraph enum's values, so transformed or curated enums stay
generated locally. Since the API lives on enum types, non-enum external
references are impossible by construction rather than needing validation.

Author: jwils

config/site/support/doctest_helper.rb

@@ -107,10 +107,12 @@ module ElasticGraph
       extend ::RSpec::Mocks::ExampleMethods
 
       # The examples source enum values from an app-defined proto enum class; provide one here.
-      proto_enum_entry = ::Data.define(:name)
+      proto_enum_entry = ::Data.define(:name, :number)
       currency_proto_enum = ::Class.new
       currency_proto_enum.define_singleton_method(:enums) do
-        [:CURRENCY_UNKNOWN_DO_NOT_USE, :CURRENCY_USD, :CURRENCY_CAD].map { |name| proto_enum_entry.new(name: name) }
+        [:CURRENCY_UNKNOWN_DO_NOT_USE, :CURRENCY_USD, :CURRENCY_CAD].each_with_index.map do |name, number|
+          proto_enum_entry.new(name: name, number: number)
+        end
       end
 
       stub_const("MyApp::Protos::Currency", currency_proto_enum)

elasticgraph-proto_ingestion/README.md

@@ -219,12 +219,14 @@ In the examples below, this stand-in plays the role of your app's generated prot
 module MyApp
   module Protos
     class Currency
-      EnumEntry = ::Data.define(:name)
+      EnumEntry = ::Data.define(:name, :number)
 
       def self.enums
-        [:CURRENCY_UNKNOWN_DO_NOT_USE, :CURRENCY_USD, :CURRENCY_CAD].map do |name|
-          EnumEntry.new(name: name)
-        end
+        [
+          EnumEntry.new(name: :CURRENCY_UNKNOWN_DO_NOT_USE, number: 0),
+          EnumEntry.new(name: :CURRENCY_USD, number: 1),
+          EnumEntry.new(name: :CURRENCY_CAD, number: 2)
+        ]
       end
     end
   end
@@ -254,6 +256,33 @@ When an enum has one or more external sources, `elasticgraph-proto_ingestion` us
 as the source of the generated enum's values. When multiple sources are registered for the
 same enum, they must all resolve to the same value set.
 
+### Referencing Existing Protobuf Types
+
+For enums that exactly match a canonical proto enum, you can go further and reference the
+existing proto type instead of generating a duplicate local enum. Pass `proto:` and `import:`
+to `external_proto_enum`, and `schema.proto` will import the named file and use the external
+type name directly:
+
+```ruby
+# in config/schema/currency.rb
+
+ElasticGraph.define_schema do |schema|
+  schema.enum_type "Currency" do |t|
+    t.values "USD", "CAD"
+    t.external_proto_enum MyApp::Protos::Currency,
+      proto: "myapp.types.Currency",
+      import: "myapp/types/currency.proto"
+  end
+end
+```
+
+A referenced enum must have exactly one option-free source whose values match the
+ElasticGraph enum's values; transformed, curated, or multi-source enums stay generated
+locally so value curation remains explicit. The source's enum entries must also expose
+`.number`, and those numbers must agree with any numbers previously pinned in
+`proto_field_numbers.yaml` — otherwise switching to the external type would silently
+reinterpret existing wire data.
+
 ## Type Mappings
 
 The generated `schema.proto` uses these built-in scalar mappings:

elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema.rb

@@ -157,6 +157,13 @@ def enum_value_numbers_for(enum_name, value_names)
           end
         end
 
+        # Returns previously pinned numbers for a protobuf enum.
+        #
+        # @api private
+        def pinned_enum_value_numbers(enum_name)
+          @proto_enum_value_numbers_by_enum[enum_name] || {}
+        end
+
         # Returns the label for a protobuf field under the configured syntax.
         #
         # @api private

elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema_elements/enum_type_extension.rb

@@ -21,15 +21,33 @@ module EnumTypeExtension
           # Describes an external proto enum registered as the source of this enum's generated values.
           ExternalProtoEnumSource = ::Data.define(:proto_enum, :exclusions, :expected_extras, :name_transform)
 
-          # Sources this enum's generated proto values from an existing proto enum class.
+          # Describes an existing proto enum type referenced instead of generating a local definition.
+          ExternalProtoEnumReference = ::Data.define(:proto_name, :import)
+
+          # Sources this enum's generated proto values from an existing proto enum class. Passing
+          # `proto:` and `import:` references that enum directly instead of generating a local enum.
           #
           # @return [void]
-          def external_proto_enum(proto_enum, exclusions: [], expected_extras: [], name_transform: nil)
+          def external_proto_enum(proto_enum, exclusions: [], expected_extras: [], name_transform: nil, proto: nil, import: nil)
             unless proto_enum.respond_to?(:enums)
               raise Errors::SchemaError, "`external_proto_enum` on `#{name}` must be given a proto enum class with `.enums`, " \
                 "but got: #{proto_enum.inspect}."
             end
 
+            if proto || import
+              unless proto.is_a?(String) && !proto.empty? && import.is_a?(String) && !import.empty?
+                raise Errors::SchemaError, "`external_proto_enum` on `#{name}` must be given both `proto` and `import` " \
+                  "as non-empty Strings to reference an external proto enum type."
+              end
+              unless exclusions.empty? && expected_extras.empty? && name_transform.nil?
+                raise Errors::SchemaError, "`external_proto_enum` on `#{name}` cannot combine `proto`/`import` with " \
+                  "`exclusions`, `expected_extras`, or `name_transform`; transformed or curated enums must stay generated locally."
+              end
+
+              @external_proto_reference = ExternalProtoEnumReference.new(proto_name: proto, import: import)
+              @proto_name = nil
+            end
+
             external_proto_enum_sources << ExternalProtoEnumSource.new(
               proto_enum: proto_enum,
               exclusions: exclusions.map(&:to_s),
@@ -46,6 +64,12 @@ def external_proto_enum_sources
             @external_proto_enum_sources ||= []
           end
 
+          # The external proto enum type referenced instead of generating a local enum, if configured.
+          #
+          # @dynamic external_proto_reference
+          # @return [ExternalProtoEnumReference, nil]
+          attr_reader :external_proto_reference
+
           # Defines an enum value and immediately validates its protobuf name.
           #
           # @return [void]
@@ -70,6 +94,11 @@ def value(value_name, &block)
           #
           # @return [String]
           def to_proto(schema)
+            if external_proto_reference
+              validate_external_proto_reference(schema)
+              return nil
+            end
+
             render_proto_enum(schema)
           end
 
@@ -91,7 +120,14 @@ def referenced_proto_types
           #
           # @return [String]
           def proto_name
-            @proto_name ||= Identifier.enum_name(name)
+            @proto_name ||= external_proto_reference&.proto_name || Identifier.enum_name(name)
+          end
+
+          # Returns the proto file imported for an externally referenced enum.
+          #
+          # @return [String, nil]
+          def protobuf_import
+            external_proto_reference&.import
           end
 
           # @private
@@ -161,11 +197,57 @@ def proto_enum_value_names
 
           def enum_value_names_from_source(source)
             name_transform = source.name_transform || :itself.to_proc
-            mapped_values = source.proto_enum.enums.map { |enum_entry| name_transform.call(enum_entry.name.to_s).to_s }
+            mapped_values = source.proto_enum.enums.map { |entry| name_transform.call(entry.name.to_s).to_s }
             (mapped_values - source.exclusions + source.expected_extras).uniq
           rescue => e
             raise Errors::SchemaError, "Failed loading external proto enum values for `#{name}` from `#{source.proto_enum}`: #{e.message}"
           end
+
+          def validate_external_proto_reference(schema)
+            unless external_proto_enum_sources.one?
+              raise Errors::SchemaError, "External proto enum `#{name}` must use exactly one `external_proto_enum` " \
+                "source; multi-source enums cannot be safely referenced externally."
+            end
+
+            numbers_by_value_name = enum_value_numbers_from_source(external_proto_enum_sources.first)
+            external_names = numbers_by_value_name.keys.sort
+            elasticgraph_names = values_by_name.keys.map(&:to_s).uniq.sort
+            if external_names != elasticgraph_names
+              raise Errors::SchemaError, "External proto enum `#{name}` values do not match the ElasticGraph enum values. " \
+                "External values: #{external_names.join(", ")}. ElasticGraph values: #{elasticgraph_names.join(", ")}."
+            end
+
+            schema.pinned_enum_value_numbers(Identifier.enum_name(name)).each do |value_name, pinned_number|
+              external_number = numbers_by_value_name[value_name]
+              if external_number && external_number != pinned_number
+                raise Errors::SchemaError, "External proto enum `#{name}` assigns `#{value_name}` the number " \
+                  "#{external_number}, but previously dumped artifacts pin it to #{pinned_number}; referencing it would " \
+                  "silently reinterpret existing wire data."
+              end
+
+              conflicting_name = numbers_by_value_name.find do |external_name, number|
+                external_name != value_name && number == pinned_number
+              end&.first
+              if conflicting_name
+                raise Errors::SchemaError, "External proto enum `#{name}` assigns `#{conflicting_name}` the number " \
+                  "#{pinned_number}, which previously dumped artifacts pin to `#{value_name}`; referencing it would " \
+                  "silently reinterpret existing wire data."
+              end
+            end
+          end
+
+          def enum_value_numbers_from_source(source)
+            entries = source.proto_enum.enums
+            unless entries.all? { |entry| entry.respond_to?(:number) }
+              raise Errors::SchemaError, "External proto enum `#{name}` cannot be referenced: its enum entries " \
+                "must expose `.number` so its values can be verified against previously pinned numbers."
+            end
+            entries.to_h { |entry| [entry.name.to_s, entry.number] }
+          rescue Errors::SchemaError
+            raise
+          rescue => e
+            raise Errors::SchemaError, "Failed loading external proto enum values for `#{name}` from `#{source.proto_enum}`: #{e.message}"
+          end
         end
       end
     end

elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema.rbs

@@ -45,6 +45,7 @@ module ElasticGraph
           name_in_index: ::String
         ) -> ::Integer
         def enum_value_numbers_for: (::String, ::Array[::String]) -> ::Hash[::String, ::Integer]
+        def pinned_enum_value_numbers: (::String) -> ::Hash[::String, ::Integer]
         def field_label: (bool) -> ::String?
 
         private

elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema_elements/enum_type_extension.rbs

@@ -17,14 +17,24 @@ module ElasticGraph
             ) -> instance
           end
 
+          class ExternalProtoEnumReference
+            attr_reader proto_name: ::String
+            attr_reader import: ::String
+
+            def self.new: (proto_name: ::String, import: ::String) -> instance
+          end
+
           @proto_name: ::String?
           @external_proto_enum_sources: ::Array[ExternalProtoEnumSource]?
+          @external_proto_reference: ExternalProtoEnumReference?
 
-          def external_proto_enum: (untyped, ?exclusions: ::Array[::String | ::Symbol], ?expected_extras: ::Array[::String | ::Symbol], ?name_transform: untyped) -> void
+          def external_proto_enum: (untyped, ?exclusions: ::Array[::String | ::Symbol], ?expected_extras: ::Array[::String | ::Symbol], ?name_transform: untyped, ?proto: ::String?, ?import: ::String?) -> void
           def external_proto_enum_sources: () -> ::Array[ExternalProtoEnumSource]
+          attr_reader external_proto_reference: ExternalProtoEnumReference?
           def value: (::String) ?{ (::ElasticGraph::SchemaDefinition::SchemaElements::EnumValue & EnumValueExtension) -> void } -> void
           def proto_name: () -> ::String
-          def to_proto: (Schema) -> ::String
+          def protobuf_import: () -> ::String?
+          def to_proto: (Schema) -> ::String?
           def proto_definition_kind: () -> :enum
           def referenced_proto_types: () -> ::Array[::ElasticGraph::SchemaDefinition::SchemaElements::graphQLType]
           def configure_derived_scalar_type: (::ElasticGraph::SchemaDefinition::SchemaElements::ScalarType) -> void
@@ -36,6 +46,8 @@ module ElasticGraph
           def proto_enum_value_name: (::String) -> ::String
           def proto_enum_value_names: () -> ::Array[::String]
           def enum_value_names_from_source: (ExternalProtoEnumSource) -> ::Array[::String]
+          def validate_external_proto_reference: (Schema) -> void
+          def enum_value_numbers_from_source: (ExternalProtoEnumSource) -> ::Hash[::String, ::Integer]
         end
       end
     end