block · jwils · Jun 30, 2026 · Aug 19, 2026 · Sep 5, 2026 · Sep 5, 2026
@@ -107,7 +107,7 @@ events = records_by_type.flat_map do |type_name, records|
     {
       __typename: type_name,
       __version: 1,
-      __json_schema_version: 1
+      __schema_version: 1
     }.merge(record)
   end
 

@@ -54,7 +54,7 @@ def configure_index(index)
               "created_at" => "2019-06-02T12:00:00Z",
               "__typename" => "Widget",
               "__version" => 1,
-              "__json_schema_version" => 1
+              "__schema_version" => 1
             }
             index_name_for_writes = index_definition.index_name_for_writes(record)
             derive_index_from_template(record, datastore_core)

@@ -315,7 +315,7 @@ def update_enum_values_in(data, json_schema_defs, type_name)
         else
           props = json_schema_def.fetch("properties")
           data.to_h do |field_name, field_value|
-            unless [:__version, :__typename, :__json_schema_version].include?(field_name)
+            unless [:__version, :__typename, :__schema_version].include?(field_name)
               field_type = props.fetch(word_to_snake_case(field_name.to_s)).fetch("ElasticGraph").fetch("type")[/\w+/]
               field_value = update_enum_values_in(field_value, json_schema_defs, field_type)
             end

@@ -89,3 +89,23 @@ module MyCompany
   end
 end
 ```
+
+A decoded event hash may carry a `schema_version` to request a specific schema artifact version. The
+key is optional, because an ingestion format may have no versions at all. Each ingestion adapter
+decides what a missing version means for its own format. `elasticgraph-json_ingestion` uses the latest
+available JSON schema version.
+
+Decoders identify their format with an `ingestion_format` key (for example, `"json"`). Adapters use
+that tag to route events independently of whether the format has schema versions. Untagged events
+remain compatible with JSON callers; other formats must supply a tag. A sole adapter receives all
+untagged events so it can provide detailed validation errors, but must still recognize an explicit tag.
+
+A successful ingestion adapter result supplies both a record preparer and a normalized event through
+`IngestionAdapter::ValidationResult.valid(record_preparer, event: normalized_event)`. The normalized
+event must preserve the event's identity, record, and transport metadata without mutating the input.
+For versioned formats, set `schema_version` to the artifact version actually selected for validation
+and preparation. Formats without versions may omit it. Operations, latency logs, and warehouse
+partitions use this normalized envelope.
+
+See [the JSON ingestion upgrade guide](../elasticgraph-json_ingestion/README.md#upgrading-an-existing-json-deployment)
+when upgrading an existing deployment.
@@ -47,11 +47,11 @@ class Config < Support::Config.define(:latency_slo_thresholds_by_timestamp_in_ms
           },
           skip_record_validation_percents_by_type: {
             description: "Map of GraphQL type names to the percentage of records of that type whose per-record " \
-              "JSON schema validation should be skipped. `0` (or an absent key) validates every record of the " \
+              "schema validation should be skipped. `0` (or an absent key) validates every record of the " \
               "type; `100` skips every record; values in between sample, and may be fractional. The decision is " \
               "deterministic per event id (`type:id@vversion`), so the same event makes the same choice on every " \
-              "retry and on every indexer pod. The event envelope (op, id, type, version, json_schema_version, " \
-              "latency_timestamps) is always validated, regardless of this setting.\n\n" \
+              "retry and on every indexer pod. The ingestion adapter always validates the event envelope, " \
+              "including any format-specific schema version, regardless of this setting.\n\n" \
               "With a large schema the per-record schema walk consumes a significant share of indexing CPU: every " \
               "record is checked against every regex, enum, min/max, format, and abstract-type discriminator " \
               "defined for its type. Skipping it trades that check for throughput, which is worthwhile when " \

@@ -21,8 +21,13 @@ def initialize(config:, schema_artifacts:, logger:)
           # must be defined, but nothing to do
         end
 
+        # Tag each decoded event with `ingestion_format` so that adapters can recognize their format.
+        # The tag is independent of schema version; untagged events default to JSON when JSON is available.
+        #
         # @param payload [String] a raw payload from the transport
-        # @return [Array<Hash<String, Object>>] the decoded ElasticGraph indexing events
+        # @return [Array<Hash<String, Object>>] the decoded ElasticGraph indexing events. An event may
+        #   include a `schema_version`, but does not have to: an ingestion format with no versions
+        #   omits it. Each ingestion adapter decides what a missing version means for its own format.
         def decode(payload)
           # :nocov: -- must return an array to satisfy Steep type checking but never called
           []

@@ -23,7 +23,7 @@ def initialize(schema_artifacts:, logger:)
 
         # Indicates whether this adapter recognizes the given event as one of its own. When multiple
         # adapters are available, the indexer routes each event to the first adapter that returns
-        # `true`. (When exactly one adapter is available, it receives all events.)
+        # `true`. (When exactly one adapter is available, it receives all untagged events.)
         #
         # @param event [Hash<String, Object>] an ElasticGraph indexing event
         # @return [Boolean] whether this adapter handles the event
@@ -34,14 +34,19 @@ def handles_event?(event)
         end
 
         # Validates the given event and resolves the record preparer appropriate for the event's
-        # schema version.
+        # schema version. The successful result carries a normalized copy of the event, including
+        # the selected `schema_version` when the format is versioned. Preserve event identity,
+        # record data, and transport metadata; do not mutate the caller's event.
+        #
+        # The event's `schema_version` is optional, because an ingestion format may have no versions
+        # at all. Each adapter decides what a missing version means for its own format.
         #
         # @param event [Hash<String, Object>] an ElasticGraph indexing event
         # @param skip_record_validation [Boolean] whether to skip record validation; the event envelope must still be validated
         # @return [ValidationResult] the result of validating the event
         def validate_event(event, skip_record_validation: false)
           # simplecov:disable -- must return a result to satisfy Steep type checking but never called
-          ValidationResult.valid(RecordPreparer::Identity)
+          ValidationResult.valid(RecordPreparer::Identity, event: event)
           # simplecov:enable
         end
       end
@@ -55,22 +60,25 @@ def validate_event(event, skip_record_validation: false)
       Failure = ::Data.define(:payload_description, :message)
 
       # Returned by {Interface#validate_event}. Either `failure` is non-nil (the event was invalid)
-      # or `record_preparer` is non-nil (the event was valid and its record can be prepared for
+      # or `event` and `record_preparer` are non-nil (the event was valid and its record can be prepared for
       # indexing with the given preparer).
       #
+      # @!attribute [r] event
+      #   @return [Hash<String, Object>, nil] normalized event, when validation succeeds
       # @!attribute [r] record_preparer
       #   @return [Object, nil] preparer for the event's record, when the event is valid
       # @!attribute [r] failure
       #   @return [Failure, nil] description of the validation problem, when the event is invalid
-      ValidationResult = ::Data.define(:record_preparer, :failure) do
+      ValidationResult = ::Data.define(:event, :record_preparer, :failure) do
         # @implements ValidationResult
 
         # Builds a result for a valid event.
         #
+        # @param event [Hash<String, Object>] normalized event
         # @param record_preparer [Object] preparer for the event's record
         # @return [ValidationResult]
-        def self.valid(record_preparer)
-          new(record_preparer: record_preparer, failure: nil)
+        def self.valid(record_preparer, event:)
+          new(event: event, record_preparer: record_preparer, failure: nil)
         end
 
         # Builds a result for an invalid event.
@@ -79,7 +87,7 @@ def self.valid(record_preparer)
         # @param message [String] detailed validation failure message
         # @return [ValidationResult]
         def self.invalid(payload_description:, message:)
-          new(record_preparer: nil, failure: Failure.new(payload_description: payload_description, message: message))
+          new(event: nil, record_preparer: nil, failure: Failure.new(payload_description: payload_description, message: message))
         end
       end
     end

@@ -6,6 +6,7 @@
 #
 # frozen_string_literal: true
 
+require "elastic_graph/constants"
 require "elastic_graph/indexer/event_id"
 require "elastic_graph/indexer/failed_event_error"
 require "elastic_graph/indexer/operation/update"
@@ -38,6 +39,7 @@ def build(event)
             return build_failed_result(event, failure.payload_description, failure.message)
           end
 
+          event = validation_result.event # : event
           record_preparer = validation_result.record_preparer # : _RecordPreparer
           if skip_record_validation
             build_success_result_isolating_malformed_records(event, record_preparer, adapter)
@@ -75,10 +77,10 @@ def build_success_result_isolating_malformed_records(event, record_preparer, ada
         end
 
         # Routes the event to the first ingestion adapter that recognizes it. When exactly one
-        # adapter is available, it receives all events--including unrecognizable ones--so that
+        # adapter is available, it receives all untagged events--including unrecognizable ones--so that
         # its more specific validation failure messages are used.
         def ingestion_adapter_for(event)
-          return ingestion_adapters.first if ingestion_adapters.one?
+          return ingestion_adapters.first if ingestion_adapters.one? && !event.key?(INGESTION_FORMAT_KEY)
 
           ingestion_adapters.find { |adapter| adapter.handles_event?(event) }
         end

@@ -41,21 +41,24 @@ def process(events, refresh_indices: false)
       # Like `process`, but returns failures instead of raising an exception.
       # The caller is responsible for handling the failures.
       def process_returning_failures(events, refresh_indices: false)
-        factory_results_by_event = events.to_h { |event| [event, @operation_factory.build(event)] }
-
-        factory_results = factory_results_by_event.values
+        factory_results = events.uniq.map { |event| @operation_factory.build(event) }
+        operations = factory_results.flat_map(&:operations)
 
         log_skipped_record_validations(factory_results)
 
-        bulk_result = @datastore_router.bulk(factory_results.flat_map(&:operations), refresh: refresh_indices)
+        bulk_result = @datastore_router.bulk(operations, refresh: refresh_indices)
         successful_operations = bulk_result.successful_operations(check_failures: false)
 
         calculate_latency_metrics(successful_operations, bulk_result.noop_results)
 
+        # Adapters may normalize event envelopes, so correlate failures using the events on the
+        # operations rather than the original input hashes.
+        operations_by_event = operations.group_by(&:event)
+
         all_failures =
           factory_results.map(&:failed_event_error).compact +
           bulk_result.failure_results.map do |result|
-            all_operations_for_event = factory_results_by_event.fetch(result.event).operations
+            all_operations_for_event = operations_by_event.fetch(result.event)
             FailedEventError.from_failed_operation_result(result, all_operations_for_event.to_set)
           end
 
@@ -142,12 +145,18 @@ def calculate_latency_metrics(successful_operations, noop_results)
 
           result = successful_events.include?(event) ? "success" : "noop"
 
+          # The schema version is optional, since an ingestion format may have no versions at all.
+          schema_version = event[SCHEMA_VERSION_KEY]
+
           @logger.info({
             "message_type" => "ElasticGraphIndexingLatencies",
             "message_id" => event["message_id"],
             "event_type" => event.fetch("type"),
             "event_id" => EventID.from_event(event).to_s,
-            JSON_SCHEMA_VERSION_KEY => event.fetch(JSON_SCHEMA_VERSION_KEY),
+            SCHEMA_VERSION_KEY => schema_version,
+            # Deprecated alias of `schema_version`, kept so that dashboards and monitors that watch
+            # the old name keep working.
+            JSON_SCHEMA_VERSION_KEY => schema_version,
             "latencies_in_ms_from" => latencies_in_ms_from,
             "slo_results" => slo_results,
             "result" => result

@@ -14,16 +14,28 @@ module ElasticGraph
   class Indexer
     module TestSupport
       module Converters
+        # Attributes that describe the event rather than the record, so they never reach the record.
+        # `__json_schema_version` is the legacy name of `__schema_version`; projects generated before
+        # the schema version became ingestion-format-neutral still use it.
+        EVENT_ONLY_ATTRIBUTES = ["__typename", "__version", "__schema_version", "__json_schema_version"]
+
         # Helper method for testing and generating fake data to convert a factory record into an event
         def self.upsert_event_for(record)
-          {
+          event = {
             "op" => "upsert",
             "id" => record.fetch("id"),
             "type" => record.fetch("__typename"),
             "version" => record.fetch("__version"),
-            "record" => record.except("__typename", "__version", "__json_schema_version"),
-            JSON_SCHEMA_VERSION_KEY => record.fetch("__json_schema_version")
+            "record" => record.except(*EVENT_ONLY_ATTRIBUTES)
           }
+
+          # The schema version is optional, so include it only when the factory supplies one.
+          schema_version = record.fetch("__schema_version") { record["__json_schema_version"] }
+          unless schema_version.nil?
+            event[SCHEMA_VERSION_KEY] = schema_version
+          end
+
+          event
         end
 
         # Helper method to create an array of events given an array of records

@@ -25,16 +25,17 @@ module ElasticGraph
       end
 
       class ValidationResultSupertype < ::Data
+        attr_reader event: event?
         attr_reader record_preparer: _RecordPreparer?
         attr_reader failure: Failure?
 
-        def initialize: (record_preparer: _RecordPreparer?, failure: Failure?) -> void
-        def self.new: (record_preparer: _RecordPreparer?, failure: Failure?) -> instance
+        def initialize: (event: event?, record_preparer: _RecordPreparer?, failure: Failure?) -> void
+        def self.new: (event: event?, record_preparer: _RecordPreparer?, failure: Failure?) -> instance
         def self.members: () -> ::Array[::Symbol]
       end
 
       class ValidationResult < ValidationResultSupertype
-        def self.valid: (_RecordPreparer) -> ValidationResult
+        def self.valid: (_RecordPreparer, event: event) -> ValidationResult
         def self.invalid: (payload_description: ::String, message: ::String) -> ValidationResult
       end
     end

@@ -2,6 +2,8 @@ module ElasticGraph
   class Indexer
     module TestSupport
       module Converters
+        EVENT_ONLY_ATTRIBUTES: ::Array[::String]
+
         def self.upsert_event_for: (::Hash[::String, untyped]) -> ::Hash[::String, untyped]
 
         def self.upsert_events_for_records: (

@@ -96,7 +96,7 @@ def build_address_event_without_geolocation
       end
 
       def build_widget(json_schema_version:)
-        event = build_upsert_event(:widget, __json_schema_version: json_schema_version)
+        event = build_upsert_event(:widget, __schema_version: json_schema_version)
         event.merge("record" => (yield event.fetch("record")))
       end
     end
@@ -116,7 +116,7 @@ def build_widget(json_schema_version:)
         write_address_schema_def(json_schema_version: 2, address_extras: "t.deleted_field 'deprecated'")
         dump_artifacts
 
-        event = build_upsert_event(:address, id: "abc", deprecated: "foo", __json_schema_version: 1)
+        event = build_upsert_event(:address, id: "abc", deprecated: "foo", __schema_version: 1)
         expect(event.dig("record", "deprecated")).to eq("foo")
 
         boot_indexer.processor.process([event], refresh_indices: true)
@@ -162,8 +162,8 @@ def get_address_payload(id)
         # included at that part of the JSON schema. So here we verify that the factory includes that.
         expect(build(:team_season)).to include(__typename: "TeamSeason")
 
-        v1_event = build_upsert_event(:team, __json_schema_version: 1)
-        v2_event = build_upsert_event(:team, __json_schema_version: 2)
+        v1_event = build_upsert_event(:team, __schema_version: 1)
+        v2_event = build_upsert_event(:team, __schema_version: 2)
           .then { |event| ::JSON.generate(event) }
           # Fix the event to align with the v2 schema, since `build_upsert_event` doesn't automatically
           # know that the `__typename` should be `SeasonOfATeam` instead of `TeamSeason`.
@@ -200,8 +200,8 @@ def get_address_payload(id)
           end
           dump_artifacts
 
-          v1_event = build_upsert_event(:team, __json_schema_version: 1)
-          v2_event = build_upsert_event(:team, __json_schema_version: 2)
+          v1_event = build_upsert_event(:team, __schema_version: 1)
+          v2_event = build_upsert_event(:team, __schema_version: 2)
 
           expect {
             boot_indexer.processor.process([v1_event, v2_event], refresh_indices: true)
@@ -244,9 +244,9 @@ def get_address_payload(id)
           end
           dump_artifacts
 
-          v1_event = build_upsert_event(:team, __json_schema_version: 1)
+          v1_event = build_upsert_event(:team, __schema_version: 1)
           v1_event = ::JSON.parse(::JSON.generate(v1_event).gsub('"name":', '"full_name":'))
-          v2_event = build_upsert_event(:team, __json_schema_version: 2)
+          v2_event = build_upsert_event(:team, __schema_version: 2)
 
           expect {
             boot_indexer.processor.process([v1_event, v2_event], refresh_indices: true)
@@ -288,7 +288,7 @@ def get_address_payload(id)
           end
           dump_artifacts
 
-          v1_event = build_upsert_event(:team, __json_schema_version: 1)
+          v1_event = build_upsert_event(:team, __schema_version: 1)
 
           expect {
             boot_indexer.processor.process([v1_event], refresh_indices: true)
@@ -323,7 +323,7 @@ def get_address_payload(id)
           write_address_schema_def(json_schema_version: 2, schema_extras: 'schema.deleted_type "Team"')
           dump_artifacts
 
-          v1_event = build_upsert_event(:team, __json_schema_version: 1)
+          v1_event = build_upsert_event(:team, __schema_version: 1)
           boot_indexer.processor.process([v1_event], refresh_indices: true)
 
           expect(search_for_ids("teams")).to be_empty