Generate proto messages for non-indexed sourced_from source types

elasticgraph-proto_ingestion/README.md

@@ -77,6 +77,17 @@ end
 After running `bundle exec rake schema_artifacts:dump`, ElasticGraph will generate a `schema.proto`
 schema artifact, and will maintain a `proto_field_numbers.yaml` file alongside your schema definition.
 
+### Which Types Get a Message
+
+`schema.proto` contains a message for each type you can publish events for, and for each type they
+reference:
+
+- Indexed types. An indexed abstract type also gets a `oneof` wrapper message.
+- `sourced_from` source types. A source type needs no index of its own.
+
+Derived indexing types get no message. ElasticGraph builds their documents from the events of other
+types, so a publisher never sends one.
+
 ### Protecting Protobuf Compatibility With Buf
 
 Install the [Buf CLI](https://buf.build/docs/installation/) anywhere that dumps schema artifacts.

elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/results_extension.rb

@@ -37,10 +37,16 @@ def protobuf_schema_generator
             # runtime in {APIExtension.extended}.
             extension_state = state # : ElasticGraph::SchemaDefinition::State & StateExtension
 
+            # Resolve `sourced_from` update targets before touching `all_types` so that `sourced_from`
+            # validation errors take precedence over any errors raised while generating derived types.
+            sourced_from_source_type_names = sourced_update_targets_by_source_type_name.keys.to_set
+
             Schema.new(
               state: extension_state,
               all_types: all_types,
-              ingestion_state: extension_state.proto_ingestion_state
+              ingestion_state: extension_state.proto_ingestion_state,
+              sourced_from_source_type_names: sourced_from_source_type_names,
+              derived_indexing_type_names: derived_indexing_type_names
             )
           end
         end

elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema.rb

@@ -64,9 +64,19 @@ def self.validate_header_lines(header_lines)
         # @param state [ElasticGraph::SchemaDefinition::State]
         # @param all_types [Array<ElasticGraph::SchemaDefinition::SchemaElements::graphQLType>]
         # @param ingestion_state [ProtoIngestionState] this extension's configured schema definition state
-        def initialize(state:, all_types:, ingestion_state:)
+        # @param sourced_from_source_type_names [Set<String>] names of the types that feed `sourced_from` fields
+        # @param derived_indexing_type_names [Set<String>] names of the types the indexer derives from other types
+        def initialize(
+          state:,
+          all_types:,
+          ingestion_state:,
+          sourced_from_source_type_names:,
+          derived_indexing_type_names:
+        )
           @state = state
           @all_types = all_types
+          @sourced_from_source_type_names = sourced_from_source_type_names
+          @derived_indexing_type_names = derived_indexing_type_names
           @package_name = ingestion_state.package_name
           @syntax = self.class.validate_syntax(ingestion_state.syntax)
           @header_lines = self.class.validate_header_lines(ingestion_state.header_lines)
@@ -146,10 +156,10 @@ def proto2?
 
         private
 
-        # Selects the indexed root types and every type transitively referenced by their protobuf
+        # Selects the ingestible types and every type transitively referenced by their protobuf
         # representations. All traversal state is local so repeated calls are independent.
         def proto_types
-          types_to_visit = _ = @state.indexed_types_by_index_name.values.dup
+          types_to_visit = ingestible_types
           type_names_to_render = ::Set.new
 
           while (type = types_to_visit.shift)
@@ -163,6 +173,22 @@ def proto_types
           end
         end
 
+        # The types a publisher can send events for: the indexed types, plus the `sourced_from`
+        # source types, which need no index of their own. Derived indexing types are excluded
+        # because the indexer builds their documents from the events of other types, so a publisher
+        # never sends one. This matches the set of types the JSON schema event envelope accepts,
+        # except for an abstract type: the envelope accepts each concrete subtype by name, while a
+        # proto schema also gets a `oneof` wrapper message for the abstract type itself.
+        def ingestible_types
+          source_types = @sourced_from_source_type_names.filter_map do |type_name|
+            @state.object_types_by_name[type_name]
+          end
+
+          types = @state.indexed_types_by_index_name.values + source_types
+
+          _ = types.reject { |type| @derived_indexing_type_names.include?(type.name) }
+        end
+
         def render_definitions(types)
           types
             .sort_by(&:proto_name)

elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema.rbs

@@ -16,12 +16,16 @@ module ElasticGraph
         @all_types: ::Array[::ElasticGraph::SchemaDefinition::SchemaElements::graphQLType]
         @package_name: ::String
         @field_number_mappings: FieldNumberMappings
+        @sourced_from_source_type_names: ::Set[::String]
+        @derived_indexing_type_names: ::Set[::String]
         @previous_field_names_by_type_name_and_field_name: ::Hash[::String, ::Hash[::String, ::Array[::String]]]?
 
         def initialize: (
           state: ::ElasticGraph::SchemaDefinition::State,
           all_types: ::Array[::ElasticGraph::SchemaDefinition::SchemaElements::graphQLType],
-          ingestion_state: ProtoIngestionState
+          ingestion_state: ProtoIngestionState,
+          sourced_from_source_type_names: ::Set[::String],
+          derived_indexing_type_names: ::Set[::String]
         ) -> void
 
         def to_proto: () -> ::String
@@ -44,6 +48,7 @@ module ElasticGraph
         private
 
         def proto_types: () -> ::Array[untyped]
+        def ingestible_types: () -> ::Array[untyped]
         def render_definitions: (::Array[untyped] types) -> ::String
         def render_imports: (::Array[untyped] types) -> ::Array[::String]
         def render_header_lines: () -> ::Array[::String]

elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/ingestible_types_spec.rb

@@ -0,0 +1,159 @@
+# Copyright 2024 - 2026 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+require "elastic_graph/proto_ingestion/schema_definition/api_extension"
+
+module ElasticGraph
+  module ProtoIngestion
+    module SchemaDefinition
+      # Covers which types get a message: the types a publisher can send events for, and every type
+      # they reference.
+      RSpec.describe Schema, "ingestible types" do
+        it "generates a message for a `sourced_from` source type that has no index of its own" do
+          proto = define_proto_schema do |s|
+            define_component_type(s)
+
+            s.object_type "ComponentDesign" do |t|
+              t.field "id", "ID!"
+              t.field "component_id", "ID"
+              t.field "designer_name", "String"
+            end
+          end
+
+          expect(proto_type_def_from(proto, "ComponentDesign")).to eq(<<~PROTO.strip)
+            message ComponentDesign {
+              string id = 1;
+              string component_id = 2;
+              string designer_name = 3;
+              // Next field number: 4
+            }
+          PROTO
+        end
+
+        it "generates a message for a type referenced by a non-indexed source type" do
+          proto = define_proto_schema do |s|
+            define_component_type(s)
+
+            s.object_type "ComponentDesign" do |t|
+              t.field "id", "ID!"
+              t.field "component_id", "ID"
+              t.field "designer_name", "String"
+              t.field "studio", "DesignStudio"
+            end
+
+            s.object_type "DesignStudio" do |t|
+              t.field "name", "String"
+            end
+          end
+
+          expect(proto_type_def_from(proto, "DesignStudio")).to eq(<<~PROTO.strip)
+            message DesignStudio {
+              string name = 1;
+              // Next field number: 2
+            }
+          PROTO
+        end
+
+        it "generates a oneof wrapper and subtype messages for an abstract source type" do
+          proto = define_proto_schema do |s|
+            define_component_type(s)
+
+            s.interface_type "ComponentDesign" do |t|
+              t.field "id", "ID!"
+              t.field "component_id", "ID"
+              t.field "designer_name", "String"
+            end
+
+            s.object_type "InternalComponentDesign" do |t|
+              t.implements "ComponentDesign"
+              t.field "id", "ID!"
+              t.field "component_id", "ID"
+              t.field "designer_name", "String"
+            end
+          end
+
+          expect(proto_type_def_from(proto, "ComponentDesign")).to eq(<<~PROTO.strip)
+            message ComponentDesign {
+              oneof value {
+                .elasticgraph.InternalComponentDesign internal_component_design = 1;
+              }
+              // Next field number: 2
+            }
+          PROTO
+
+          expect(proto_type_def_from(proto, "InternalComponentDesign")).to eq(<<~PROTO.strip)
+            message InternalComponentDesign {
+              string id = 1;
+              string component_id = 2;
+              string designer_name = 3;
+              // Next field number: 4
+            }
+          PROTO
+        end
+
+        it "generates no message for a derived indexing type, since no publisher sends its events" do
+          proto = define_proto_schema do |s|
+            s.object_type "Widget" do |t|
+              t.field "id", "ID!"
+              t.field "workspace_id", "ID"
+              t.field "name", "String"
+              t.index "widgets"
+
+              t.derive_indexed_type_fields "WidgetWorkspace", from_id: "workspace_id" do |derive|
+                derive.immutable_value "name", from: "name"
+              end
+            end
+
+            s.object_type "WidgetWorkspace" do |t|
+              t.field "id", "ID!"
+              t.field "name", "String"
+              t.index "widget_workspaces"
+            end
+          end
+
+          expect(proto_type_def_from(proto, "Widget")).not_to be nil
+          expect(proto_type_def_from(proto, "WidgetWorkspace")).to be nil
+        end
+
+        it "generates no message for a type that no ingestible type references" do
+          proto = define_proto_schema do |s|
+            s.object_type "Widget" do |t|
+              t.field "id", "ID!"
+              t.field "name", "String"
+              t.index "widgets"
+            end
+
+            s.object_type "Unreferenced" do |t|
+              t.field "id", "ID!"
+            end
+          end
+
+          expect(proto_type_def_from(proto, "Unreferenced")).to be nil
+        end
+
+        # Defines an indexed type with a top-level `sourced_from` field fed by `ComponentDesign`.
+        def define_component_type(schema)
+          schema.object_type "Component" do |t|
+            t.field "id", "ID!"
+            t.field "name", "String"
+
+            t.field "designer_name", "String" do |f|
+              f.sourced_from "design", "designer_name"
+            end
+
+            t.relates_to_one "design", "ComponentDesign", via: "component_id", dir: :in, indexing_only: true
+
+            t.index "components" do |i|
+              i.has_had_multiple_sources!
+            end
+          end
+        end
+      end
+    end
+  end
+end

elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/schema_spec.rb

@@ -129,7 +129,9 @@ module SchemaDefinition
           generator = Schema.new(
             state: results.state,
             all_types: results.send(:all_types),
-            ingestion_state: results.state.proto_ingestion_state
+            ingestion_state: results.state.proto_ingestion_state,
+            sourced_from_source_type_names: results.sourced_update_targets_by_source_type_name.keys.to_set,
+            derived_indexing_type_names: results.derived_indexing_type_names
           )
 
           first_generation = generator.to_proto