Add audit tables and orchestrator for batch upload v2 (#244)

* Add audit tables and orchestrator for batch upload v2

Story 4: Add chunk-level audit logs and error tracking
- Create certification_batch_upload_audit_logs table with status tracking
- Create certification_batch_upload_errors table for failed records
- Add audit_logs and upload_errors associations to CertificationBatchUpload
- Create-then-update pattern: audit logs start as "started", update to "completed"/"failed"
- Duration calculated from timestamps (created_at to updated_at)
- Clear distinction: record failures (validation) vs chunk failures (system errors)

Story 5: Create CertificationBatchUploadOrchestrator
- Single entry point for all upload sources (UI, API, storage events)
- Validates file exists in cloud storage before creating record
- Enqueues ProcessCertificationBatchUploadJob for async processing
- Returns batch upload record for status tracking

Design decisions:
- Errors belong to batch upload (not chunks) - user-facing, not implementation detail
- Audit logs track chunk-level outcomes with aggregated counts
- Status enum: started → completed (success) or failed (system error)
- Test helpers in spec/support/batch_upload_helpers.rb

Resolves #203, #204

* Add strict_loading to audit_logs and upload_errors associations

Author: giverm

reporting-app/app/models/certification_batch_upload.rb

@@ -16,6 +16,14 @@ class CertificationBatchUpload < ApplicationRecord
 
   belongs_to :uploader, class_name: "User"
   has_one_attached :file
+  has_many :audit_logs,
+           class_name: "CertificationBatchUploadAuditLog",
+           strict_loading: true,
+           dependent: :destroy
+  has_many :upload_errors,
+           class_name: "CertificationBatchUploadError",
+           strict_loading: true,
+           dependent: :destroy
 
   validates :filename, presence: true
   validate :file_or_storage_key_present, on: :create

reporting-app/app/models/certification_batch_upload_audit_log.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+class CertificationBatchUploadAuditLog < ApplicationRecord
+  belongs_to :certification_batch_upload
+
+  attribute :status, :string, default: "started"
+  enum :status, {
+    started: "started",      # Chunk processing began
+    completed: "completed",  # Chunk succeeded (individual records may have failed validation)
+    failed: "failed"         # Chunk job crashed (system error)
+  }
+
+  validates :chunk_number, presence: true, numericality: { greater_than: 0 }
+  validates :succeeded_count, numericality: { greater_than_or_equal_to: 0 }
+  validates :failed_count, numericality: { greater_than_or_equal_to: 0 }
+end

reporting-app/app/models/certification_batch_upload_error.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+class CertificationBatchUploadError < ApplicationRecord
+  belongs_to :certification_batch_upload
+
+  validates :row_number, presence: true, numericality: { greater_than: 0 }
+  validates :error_code, presence: true
+  validates :error_message, presence: true
+end

reporting-app/app/services/certification_batch_upload_orchestrator.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+class CertificationBatchUploadOrchestrator
+  class FileNotFoundError < StandardError; end
+
+  def initialize(storage_adapter: nil)
+    @storage = storage_adapter || Rails.application.config.storage_adapter
+  end
+
+  # Initiate batch upload processing
+  # @param source_type [Symbol] How file was uploaded (:ui, :api, :storage_event)
+  # @param filename [String] Original filename
+  # @param storage_key [String] Cloud storage object key
+  # @param uploader [User] User who initiated the upload
+  # @param metadata [Hash] Optional metadata (reserved for future use)
+  # @return [CertificationBatchUpload] Created batch upload record
+  # @raise [FileNotFoundError] if file doesn't exist in storage
+  def initiate(source_type:, filename:, storage_key:, uploader:, metadata: {})
+    # Validate file exists in cloud storage before creating DB record
+    unless @storage.object_exists?(key: storage_key)
+      raise FileNotFoundError, "File not found in storage: #{storage_key}"
+    end
+
+    # Create batch upload record
+    batch_upload = CertificationBatchUpload.create!(
+      source_type: source_type,
+      filename: filename,
+      storage_key: storage_key,
+      uploader: uploader,
+      status: :pending
+    )
+
+    # Enqueue processing job for async execution
+    ProcessCertificationBatchUploadJob.perform_later(batch_upload.id)
+
+    batch_upload
+  end
+end

reporting-app/db/migrate/20260130180427_create_certification_batch_upload_audit_logs.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+# Migration to support batch upload v2 chunk-level audit logging
+# Tracks processing status for each 1,000-record chunk:
+# - status: started/completed/failed (chunk-level outcome)
+# - succeeded_count/failed_count: record-level results (when completed)
+# - timestamps: created_at = start, updated_at = completion (for duration calculation)
+class CreateCertificationBatchUploadAuditLogs < ActiveRecord::Migration[7.2]
+  def change
+    create_table :certification_batch_upload_audit_logs, id: :uuid do |t|
+      t.references :certification_batch_upload, type: :uuid, foreign_key: true, null: false, index: true
+      t.integer :chunk_number, null: false
+      t.string :status, null: false, default: "started"
+      t.integer :succeeded_count, default: 0
+      t.integer :failed_count, default: 0
+      t.timestamps
+
+      t.index [ :certification_batch_upload_id, :chunk_number ],
+        name: "idx_audit_logs_on_upload_chunk"
+      t.index :status, name: "idx_audit_logs_on_status"
+    end
+  end
+end

reporting-app/db/migrate/20260130180441_create_certification_batch_upload_errors.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+# Migration to support batch upload v2 error tracking
+# Stores individual failed CSV records for analysis and retry:
+# - row_number: Line number in original CSV file (user-facing reference)
+# - error_code: Structured code for categorization (e.g., VAL_001)
+# - error_message: Human-readable description
+# - row_data: Full CSV row as JSONB for retry attempts
+class CreateCertificationBatchUploadErrors < ActiveRecord::Migration[7.2]
+  def change
+    create_table :certification_batch_upload_errors, id: :uuid do |t|
+      t.references :certification_batch_upload, type: :uuid, foreign_key: true, null: false, index: true
+      t.integer :row_number, null: false
+      t.string :error_code, null: false
+      t.string :error_message, null: false
+      t.jsonb :row_data
+      t.timestamps
+
+      t.index [ :certification_batch_upload_id, :error_code ],
+        name: "idx_upload_errors_on_upload_code"
+    end
+  end
+end

reporting-app/db/schema.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+FactoryBot.define do
+  factory :certification_batch_upload_audit_log, aliases: [ :audit_log ] do
+    association :certification_batch_upload
+    chunk_number { 1 }
+    status { :started }
+    succeeded_count { 0 }
+    failed_count { 0 }
+
+    trait :completed do
+      status { :completed }
+      succeeded_count { 1000 }
+      failed_count { 0 }
+    end
+
+    trait :failed do
+      status { :failed }
+    end
+  end
+end

reporting-app/spec/factories/certification_batch_upload_audit_logs_factory.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+FactoryBot.define do
+  factory :certification_batch_upload_error, aliases: [ :upload_error ] do
+    association :certification_batch_upload
+    row_number { 1 }
+    error_code { "VAL_001" }
+    error_message { "Missing required field" }
+    row_data { { "member_id" => "M001" } }
+  end
+end

reporting-app/spec/factories/certification_batch_upload_errors_factory.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+require "rails_helper"
+
+RSpec.describe CertificationBatchUploadAuditLog, type: :model do
+  let(:user) { create(:user) }
+  let(:batch_upload) { create(:certification_batch_upload, uploader: user, storage_key: "test-key") }
+
+  describe "validations" do
+    it "requires chunk_number" do
+      log = described_class.new(certification_batch_upload: batch_upload)
+      expect(log).not_to be_valid
+      expect(log.errors[:chunk_number]).to be_present
+    end
+
+    it "requires chunk_number to be positive" do
+      log = described_class.new(
+        certification_batch_upload: batch_upload,
+        chunk_number: 0
+      )
+      expect(log).not_to be_valid
+      expect(log.errors[:chunk_number]).to include("must be greater than 0")
+    end
+
+    it "requires succeeded_count to be non-negative" do
+      log = described_class.new(
+        certification_batch_upload: batch_upload,
+        chunk_number: 1,
+        succeeded_count: -1
+      )
+      expect(log).not_to be_valid
+    end
+
+    it "requires failed_count to be non-negative" do
+      log = described_class.new(
+        certification_batch_upload: batch_upload,
+        chunk_number: 1,
+        failed_count: -1
+      )
+      expect(log).not_to be_valid
+    end
+  end
+
+  describe "status enum" do
+    let(:log) { create(:audit_log, certification_batch_upload: batch_upload, chunk_number: 1) }
+
+    it "defaults to started" do
+      expect(log.status).to eq("started")
+      expect(log).to be_started
+    end
+
+    it "can transition to completed" do
+      log.completed!
+      expect(log).to be_completed
+    end
+
+    it "can transition to failed" do
+      log.failed!
+      expect(log).to be_failed
+    end
+  end
+
+  describe "associations" do
+    it "belongs to certification_batch_upload" do
+      log = create(:audit_log, certification_batch_upload: batch_upload, chunk_number: 1)
+      expect(log.certification_batch_upload).to eq(batch_upload)
+    end
+
+    it "is destroyed when batch_upload is destroyed" do
+      log = create(:audit_log, certification_batch_upload: batch_upload, chunk_number: 1)
+      # Eager load associations to avoid strict_loading violations
+      batch_upload_with_associations = CertificationBatchUpload.includes(:audit_logs, :upload_errors).find(batch_upload.id)
+      expect { batch_upload_with_associations.destroy }.to change(described_class, :count).by(-1)
+    end
+  end
+end