add dynamic look up of schema to kafka producer

config/features.yml

@@ -2032,6 +2032,9 @@ features:
   kafka_producer:
     actor_type: cookie_id
     description: Enables the Kafka producer for the VA.gov platform
+  kafka_producer_fetch_schema_dynamically:
+    actor_type: cookie_id
+    description: Enables the Kafka producer to fetch schema dynamically
   show_about_yellow_ribbon_program:
     actor_type: user
     description: If enabled, show additional info about the yellow ribbon program

lib/kafka/avro_producer.rb

@@ -2,13 +2,16 @@
 
 require 'avro'
 require 'kafka/producer_manager'
+require 'kafka/confluent_schema_registry'
+require 'logger'
 
 module Kafka
   class AvroProducer
-    attr_reader :producer
+    attr_reader :producer, :registry
 
     def initialize(producer: nil)
       @producer = producer || Kafka::ProducerManager.instance.producer
+      @registry = Kafka::ConfluentSchemaRegistry.new(Settings.kafka_producer.schema_registry_url)
     end
 
     def produce(topic, payload, schema_version: 1)
@@ -25,8 +28,14 @@ def produce(topic, payload, schema_version: 1)
     private
 
     def get_schema(topic, schema_version)
-      schema_path = Rails.root.join('lib', 'kafka', 'schemas', "#{topic}-value-#{schema_version}.avsc")
-      Avro::Schema.parse(File.read(schema_path))
+      if Flipper.enabled?(:kafka_producer_fetch_schema_dynamically)
+        schema_json = @registry.fetch(topic)
+      else
+        schema_path = Rails.root.join('lib', 'kafka', 'schemas', "#{topic}-value-#{schema_version}.avsc")
+        schema_json = File.read(schema_path)
+      end
+
+      Avro::Schema.parse(schema_json)
     end
 
     def encode_payload(schema, payload)

lib/kafka/confluent_schema_registry.rb

@@ -0,0 +1,120 @@
+# frozen_string_literal: true
+
+class Kafka::ConfluentSchemaRegistry
+  def initialize(url)
+    headers = {
+      'Accept' => 'application/json',
+      'User-Agent' => 'Vets.gov Agent',
+      'Content-Type' => 'application/vnd.schemaregistry.v1+json'
+    }
+
+    @connection = Faraday.new(url, headers:)
+  end
+
+  def fetch(id)
+    data = get("/schemas/ids/#{id}", idempotent: true)
+    data.fetch('schema')
+  end
+
+  def register(subject, schema)
+    data = post("/subjects/#{subject}/versions", body: { schema: schema.to_s }.to_json)
+
+    data.fetch('id')
+  end
+
+  # List all subjects
+  def subjects
+    get('/subjects', idempotent: true)
+  end
+
+  # List all versions for a subject
+  def subject_versions(subject)
+    get("/subjects/#{subject}/versions", idempotent: true)
+  end
+
+  # Get a specific version for a subject
+  def subject_version(subject, version = 'latest')
+    get("/subjects/#{subject}/versions/#{version}", idempotent: true)
+  end
+
+  # Get the subject and version for a schema id
+  def schema_subject_versions(schema_id)
+    get("/schemas/ids/#{schema_id}/versions", idempotent: true)
+  end
+
+  # Check if a schema exists. Returns nil if not found.
+  def check(subject, schema)
+    data = post("/subjects/#{subject}",
+                expects: [200, 404],
+                body: { schema: schema.to_s }.to_json,
+                idempotent: true)
+    data unless data.key?('error_code')
+  end
+
+  # Check if a schema is compatible with the stored version.
+  # Returns:
+  # - true if compatible
+  # - nil if the subject or version does not exist
+  # - false if incompatible
+  # http://docs.confluent.io/3.1.2/schema-registry/docs/api.html#compatibility
+  def compatible?(subject, schema, version = 'latest')
+    data = post("/compatibility/subjects/#{subject}/versions/#{version}",
+                expects: [200, 404], body: { schema: schema.to_s }.to_json, idempotent: true)
+    data.fetch('is_compatible', false) unless data.key?('error_code')
+  end
+
+  # Check for specific schema compatibility issues
+  # Returns:
+  # - nil if the subject or version does not exist
+  # - a list of compatibility issues
+  # https://docs.confluent.io/platform/current/schema-registry/develop/api.html#sr-api-compatibility
+  def compatibility_issues(subject, schema, version = 'latest')
+    data = post("/compatibility/subjects/#{subject}/versions/#{version}",
+                expects: [200, 404], body: { schema: schema.to_s }.to_json, query: { verbose: true }, idempotent: true)
+
+    data.fetch('messages', []) unless data.key?('error_code')
+  end
+
+  # Get global config
+  def global_config
+    get('/config', idempotent: true)
+  end
+
+  # Update global config
+  def update_global_config(config)
+    put('/config', body: config.to_json, idempotent: true)
+  end
+
+  # Get config for subject
+  def subject_config(subject)
+    get("/config/#{subject}", idempotent: true)
+  end
+
+  # Update config for subject
+  def update_subject_config(subject, config)
+    put("/config/#{subject}", body: config.to_json, idempotent: true)
+  end
+
+  private
+
+  def get(path, **)
+    request(path, method: :get, **)
+  end
+
+  def put(path, **)
+    request(path, method: :put, **)
+  end
+
+  def post(path, **)
+    request(path, method: :post, **)
+  end
+
+  def request(path, method: :get, **options)
+    options = { expects: 200 }.merge!(options)
+    response = @connection.send(method, path) do |req|
+      req.headers = options[:headers] if options[:headers]
+    end
+
+    JSON.parse(response.body)
+  end
+end

spec/lib/kafka/avro_producer_spec.rb

@@ -14,6 +14,7 @@
   before do
     allow(Rails).to receive(:env).and_return(ActiveSupport::StringInquirer.new('test'))
     allow(Flipper).to receive(:enabled?).with(:kafka_producer).and_return(true)
+    allow(Flipper).to receive(:enabled?).with(:kafka_producer_fetch_schema_dynamically).and_return(true)
     allow(Kafka::OauthTokenRefresher).to receive(:new).and_return(double(on_oauthbearer_token_refresh: 'token'))
   end
 
@@ -44,25 +45,51 @@
   end
 
   context 'producing a message successfully' do
+    let(:topic1_payload_value) { "\x00\x00\x00\x00\x05\x02\x06key\nvalue\x00" }
+
     before do
       Kafka::ProducerManager.instance.send(:setup_producer)
-      allow(avro_producer).to receive(:get_schema).and_return(schema)
     end
 
     after do
       # reset the client after each test
       avro_producer.producer.client.reset
     end
 
-    it 'produces a message to the specified topic' do
-      avro_producer.produce('topic-1', valid_payload)
-      avro_producer.produce('topic-1', valid_payload)
-      avro_producer.produce('topic-2', valid_payload)
+    context 'with dynamic schema registry retrieval' do
+      it 'produces a message to the specified topic' do
+        url = 'http://sandbox.lighthouse.va.gov/ves-event-bus-infra/schema-registry/subjects/submission_trace_mock_dev-value/versions/1'
+        with_settings(Settings.kafka_producer,
+                      schema_registry_url: url) do
+          VCR.use_cassette('kafka/topic1') do
+            VCR.use_cassette('kafka/topic2') do
+              avro_producer.produce('topic-1', valid_payload)
+              avro_producer.produce('topic-2', valid_payload)
+              expect(avro_producer.producer.client.messages.length).to eq(2)
+              topic_1_messages = avro_producer.producer.client.messages_for('topic-1')
+              expect(topic_1_messages.length).to eq(1)
+              expect(topic_1_messages[0][:payload]).to be_a(String)
+              expect(topic_1_messages[0][:payload]).to eq(topic1_payload_value)
+            end
+          end
+        end
+      end
+    end
 
-      expect(avro_producer.producer.client.messages.length).to eq(3)
-      topic_1_messages = avro_producer.producer.client.messages_for('topic-1')
-      expect(topic_1_messages.length).to eq(2)
-      expect(topic_1_messages[0][:payload]).to be_a(String)
+    context 'with hardcoded schema registry retrieval' do
+      before do
+        allow(Flipper).to receive(:enabled?).with(:kafka_producer_fetch_schema_dynamically).and_return(false)
+      end
+
+      it 'produces a message to the specified topic' do
+        avro_producer.produce('test', valid_payload)
+
+        expect(avro_producer.producer.client.messages.length).to eq(1)
+        topic_1_messages = avro_producer.producer.client.messages_for('test')
+        expect(topic_1_messages.length).to eq(1)
+        expect(topic_1_messages[0][:payload]).to be_a(String)
+        expect(topic_1_messages[0][:payload]).to eq(topic1_payload_value)
+      end
     end
   end
 

spec/support/vcr_cassettes/kafka/topic1.yml

@@ -0,0 +1,64 @@
+---
+http_interactions:
+- request:
+    method: get
+    uri: http://sandbox.lighthouse.va.gov/schemas/ids/topic-1
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept:
+      - application/json
+      Content-Type:
+      - application/json
+      User-Agent:
+      - Vets.gov Agent
+      Referer:
+      - https://review-instance.va.gov
+      X-Vamf-Jwt:
+      - stubbed_token
+      X-Request-Id:
+      - ''
+      Accept-Encoding:
+      - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Date:
+      - Tue, 10 Aug 2021 15:55:41 GMT
+      Content-Type:
+      - application/json
+      Transfer-Encoding:
+      - chunked
+      Server:
+      - openresty
+      X-Vamf-Version:
+      - 1.9.0
+      B3:
+      - e1836f37ef12a997-bd3cba28a9b29067-1
+      Access-Control-Allow-Headers:
+      - x-vamf-jwt
+      X-Vamf-Build:
+      - e208742
+      X-Vamf-Timestamp:
+      - '2021-08-03T18:59:01+0000'
+      Access-Control-Allow-Origin:
+      - "*"
+      Access-Control-Allow-Methods:
+      - GET,OPTIONS
+      Access-Control-Max-Age:
+      - '3600'
+      Strict-Transport-Security:
+      - max-age=63072000; includeSubDomains; preload
+    body:
+      encoding: UTF-8
+      string: '{
+                "subject": "submission_trace_mock_dev-value",
+                "version": 1,
+                "id": 5,
+                "schema": "{\"type\":\"record\",\"name\":\"TestRecord\",\"namespace\":\"gov.va.eventbus.test.data\",\"fields\":[{\"name\":\"data\",\"type\":{\"type\":\"map\",\"values\":\"string\"},\"default\":{}}]}"
+              }'
+  recorded_at: Tue, 10 Aug 2021 15:55:41 GMT
+recorded_with: VCR 6.0.0

spec/support/vcr_cassettes/kafka/topic2.yml

@@ -0,0 +1,64 @@
+---
+http_interactions:
+- request:
+    method: get
+    uri: http://sandbox.lighthouse.va.gov/schemas/ids/topic-2
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept:
+      - application/json
+      Content-Type:
+      - application/json
+      User-Agent:
+      - Vets.gov Agent
+      Referer:
+      - https://review-instance.va.gov
+      X-Vamf-Jwt:
+      - stubbed_token
+      X-Request-Id:
+      - ''
+      Accept-Encoding:
+      - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Date:
+      - Tue, 10 Aug 2021 15:55:41 GMT
+      Content-Type:
+      - application/json
+      Transfer-Encoding:
+      - chunked
+      Server:
+      - openresty
+      X-Vamf-Version:
+      - 1.9.0
+      B3:
+      - e1836f37ef12a997-bd3cba28a9b29067-1
+      Access-Control-Allow-Headers:
+      - x-vamf-jwt
+      X-Vamf-Build:
+      - e208742
+      X-Vamf-Timestamp:
+      - '2021-08-03T18:59:01+0000'
+      Access-Control-Allow-Origin:
+      - "*"
+      Access-Control-Allow-Methods:
+      - GET,OPTIONS
+      Access-Control-Max-Age:
+      - '3600'
+      Strict-Transport-Security:
+      - max-age=63072000; includeSubDomains; preload
+    body:
+      encoding: UTF-8
+      string: '{
+                "subject": "submission_trace_mock_dev-value",
+                "version": 1,
+                "id": 6,
+                "schema": "{\"type\":\"record\",\"name\":\"TestRecord2\",\"namespace\":\"gov.va.eventbus.test.data\",\"fields\":[{\"name\":\"data\",\"type\":{\"type\":\"map\",\"values\":\"string\"},\"default\":{}}]}"
+              }'
+  recorded_at: Tue, 10 Aug 2021 15:55:41 GMT
+recorded_with: VCR 6.0.0