Edm 653/lcpe cache logic (#1314)

* preload dataset

* update to sql context

* Create preload dataset and implement logic

* Migrations

* Revert migrations

* schema

* Update sample program file with ojt_app_type column

* Revert "Update sample program file with ojt_app_type column"

This reverts commit 27ed455.

* Implement preload datasets

* Update enriched id

* Refactor

* Add period

* Remove period

* revert Gemfile.lock

* linting

* Update specs

---------

Co-authored-by: Vanson Samuel <[email protected]>

app/controllers/v1/lcpe/exams_controller.rb

@@ -1,8 +1,10 @@
 # frozen_string_literal: true
 
-class V1::Lcpe::ExamsController < ApiController
+class V1::Lcpe::ExamsController < V1::LcpeBaseController
+  before_action :validate_preload_version, only: :show
+
   def index
-    results = Lcpe::Exam.with_enriched_id
+    results = preload_dataset || Lcpe::Exam.with_enriched_id
 
     render json: results, each_serializer: Lcpe::ExamSerializer, adapter: :json, action: 'index'
   end

app/controllers/v1/lcpe/lacs_controller.rb

@@ -1,10 +1,12 @@
 # frozen_string_literal: true
 
-class V1::Lcpe::LacsController < ApiController
+class V1::Lcpe::LacsController < V1::LcpeBaseController
+  before_action :validate_preload_version, only: :show
+
   def index
     render(
       {
-        json: list,
+        json: preload_dataset || list,
         each_serializer: Lcpe::LacSerializer,
         adapter: :json,
         action: 'index'

app/controllers/v1/lcpe_base_controller.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module V1
+  class LcpeBaseController < ApiController
+    class PreloadVersionStaleError < StandardError; end
+
+    rescue_from PreloadVersionStaleError, with: :version_invalid
+
+    private
+
+    def validate_preload_version
+      preload_version = params[:id].split('@').last
+      raise PreloadVersionStaleError unless preload_version == fresh_preload.id.to_s
+    end
+
+    def preload_dataset
+      return if bypass_versioning?
+
+      set_etag(fresh_preload.id.to_s)
+      JSON.parse(fresh_preload.body)
+    end
+
+    def fresh_preload
+      @fresh_preload ||= ::Lcpe::PreloadDataset.fresh(lcpe_type)
+    end
+
+    def lcpe_type
+      "Lcpe::#{controller_name.singularize.titleize}"
+    end
+
+    def set_etag(preload_version)
+      response.set_header('ETag', preload_version)
+    end
+
+    # If additional filter params present, bypass versioning
+    def bypass_versioning?
+      scrubbed_params.present?
+    end
+
+    def scrubbed_params
+      params.except(:format, :controller, :action, controller_name.singularize.to_sym)
+    end
+
+    def version_invalid
+      render json: { error: 'Version invalid' }, status: :conflict
+    end
+  end
+end

app/models/lcpe/exam.rb

@@ -2,23 +2,21 @@
 
 module Lcpe
   class Exam < ApplicationRecord
-    REF_CODE_FN = "RIGHT(MD5(CONCAT(facility_code, '-', nexam_nm)), 5)"
-
     extend SqlContext
 
     # using Enriched IDs is a good way to ensure that
     # a stale ID preloaded from the browser is not used.
     # :nocov:
     scope :with_enriched_id, lambda {
+      preload_id = Lcpe::PreloadDataset.fresh(klass.to_s).id
       select(
         '*',
-        "#{REF_CODE_FN} AS ref_code",
-        "CONCAT(id, '@', #{REF_CODE_FN}) enriched_id"
+        "CONCAT(id, '@', #{preload_id}) enriched_id"
       )
     }
 
     scope :by_enriched_id, lambda { |enriched_id|
-      id, = enriched_id.match(/\A(\d+)@(.+)\z/).values_at(1, 2)
+      id = enriched_id.split('@').first
 
         with(enriched_query: with_enriched_id.where('id = ?', id))
           .select("#{table_name}.*", 'enriched_query.enriched_id')

app/models/lcpe/feed.rb

@@ -9,4 +9,13 @@ def self.table_name_prefix
 
     'lcpe_feed_'
   end
+
+  # Each Lcpe::Feed:: model should clarify the NORMALIZED_KLASS it's associated with
+  # For example, Lcpe::Feed::Nexam normalizes to Lcpe::Exam
+  def self.normalized_klasses
+    Lcpe::Feed.constants.map do |const|
+      feed = Lcpe::Feed.const_get(const)
+      feed.const_get(:NORMALIZED_KLASS) if feed.const_defined?(:NORMALIZED_KLASS)
+    end.compact
+  end
 end

app/models/lcpe/feed/lac.rb

@@ -13,6 +13,8 @@ class Lac < ImportableRecord
         'lc_test_fee' => { column: :fee_amt, converter: Converters::BaseConverter }
       }.freeze
 
+      NORMALIZED_KLASS = 'Lcpe::Lac'
+
       def self.normalize
         pure_sql
           .join(Lcpe::Lac.reset)

app/models/lcpe/feed/nexam.rb

@@ -14,6 +14,8 @@ class Nexam < ImportableRecord
         'nexam_fee_to_date' => { column: :end_dt, converter: Converters::BaseConverter }
       }.freeze
 
+      NORMALIZED_KLASS = 'Lcpe::Exam'
+
       def self.normalize
         pure_sql
           .join(Lcpe::Exam.reset)

app/models/lcpe/lac.rb

@@ -2,23 +2,21 @@
 
 module Lcpe
   class Lac < ApplicationRecord
-    REF_CODE_FN = "RIGHT(MD5(CONCAT(facility_code, '-', lac_nm)), 5)"
-
     extend SqlContext
 
     # using Enriched IDs is a good way to ensure that
     # a stale ID preloaded from the browser is not used.
     # :nocov:
     scope :with_enriched_id, lambda {
+      preload_id = Lcpe::PreloadDataset.fresh(klass.to_s).id
       select(
         '*',
-        "#{REF_CODE_FN} AS ref_code",
-        "CONCAT(id, '@', #{REF_CODE_FN}) enriched_id"
+        "CONCAT(id, '@', #{preload_id}) enriched_id"
       )
     }
 
     scope :by_enriched_id, lambda { |enriched_id|
-      id, = enriched_id.match(/\A(\d+)@(.+)\z/).values_at(1, 2)
+      id = enriched_id.split('@').first
 
         with(enriched_query: with_enriched_id.where('id = ?', id))
           .select("#{table_name}.*", 'enriched_query.enriched_id')

app/models/lcpe/preload_dataset.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+class Lcpe::PreloadDataset < ApplicationRecord
+  # ['Lcpe::Lac', 'Lcpe::Exam']
+  LCPE_TYPES = Lcpe::Feed.normalized_klasses.freeze
+
+  validates :subject_class, presence: true, inclusion: { in: LCPE_TYPES }
+
+  scope :of_type, ->(lcpe_type) { where(subject_class: lcpe_type).order(created_at: :desc) }
+  scope :stale, ->(lcpe_type) { of_type(lcpe_type).offset(1) }
+
+  def self.fresh(lcpe_type)
+    of_type(lcpe_type).first
+  end
+
+  def self.build(lcpe_type)
+    # wipe all exepct most recent preload for lcpe type
+    stale(lcpe_type).destroy_all
+    create(subject_class: lcpe_type).tap do |preload|
+      dataset = preload.klass.with_enriched_id
+      preload.update(body: dataset.to_json)
+    end
+  end
+
+  def klass
+    subject_class.constantize
+  end
+end

app/models/upload.rb

@@ -128,6 +128,8 @@ def normalize_lcpe!
     subject = csv_type&.constantize
 
     subject.normalize.execute
+    str_klass = subject.const_get(:NORMALIZED_KLASS)
+    Lcpe::PreloadDataset.build(str_klass)
   rescue StandardError
     nil
   end

app/serializers/lcpe/exam_serializer.rb

@@ -15,8 +15,8 @@ def json_key
 
     def serializable_hash(*)
       {
-        enriched_id: resource.enriched_id,
-        name: resource.nexam_nm
+        enriched_id: resource['enriched_id'],
+        name: resource['nexam_nm']
       }.tap(&method(:add_tests)).tap(&method(:add_institution))
     end
 

app/serializers/lcpe/lac_serializer.rb

@@ -15,10 +15,10 @@ def json_key
 
     def serializable_hash(*)
       {
-        enriched_id: resource.enriched_id,
-        lac_nm: resource.lac_nm,
-        edu_lac_type_nm: resource.edu_lac_type_nm,
-        state: resource.state
+        enriched_id: resource['enriched_id'],
+        lac_nm: resource['lac_nm'],
+        edu_lac_type_nm: resource['edu_lac_type_nm'],
+        state: resource['state']
       }.tap(&method(:add_tests)).tap(&method(:add_institution))
     end
 

lib/lcpe/sql_context.rb

@@ -5,8 +5,23 @@ module SqlContext
     class Sql
       attr_reader :gamma
 
-      def initialize(sql = nil)
-        @gamma = sql.nil? ? [] : [sql]
+      def initialize(sql = nil, &)
+        if sql.present? && block_given?
+          raise "can't provide both sql and block"
+
+        elsif sql.blank? && block_given?
+          context = :block
+          body = Proc.new
+          @gamma = [{ context:, body: }]
+
+        elsif sql.present? && !block_given?
+          context = :sql
+          body = sql
+          @gamma = [{ context:, body: }]
+
+        else # if sql.blank? && !block_given?
+          @gamma = []
+        end
       end
 
       def join(ctx)
@@ -17,10 +32,17 @@ def join(ctx)
       end
 
       def execute
-        result = gamma.join("\n")
-
         ActiveRecord::Base.transaction do
-          ActiveRecord::Base.connection.execute(result)
+          gamma.each do |item|
+            case item[:context]
+            when :sql
+              ActiveRecord::Base.connection.execute(item[:body])
+            when :block
+              item[:body].call
+            else
+              raise "invalid context: #{item[:context]}"
+            end
+          end
         end
       end
     end

spec/factories/lcpe/exams.rb

@@ -4,5 +4,9 @@
   factory :lcpe_exam, class: 'Lcpe::Exam' do
     facility_code { '57001151' }
     nexam_nm { 'AP-ADVANCED PLACEMENT EXAMS' }
+
+    trait :preloaded do
+      after(:create) { |instance| Lcpe::PreloadDataset.build(instance.class.to_s) }
+    end
   end
 end

spec/factories/lcpe/lacs.rb

@@ -6,5 +6,9 @@
     edu_lac_type_nm { 'License' }
     lac_nm { 'Gas Fitter' }
     state { 'AR' }
+
+    trait :preloaded do
+      after(:create) { |instance| Lcpe::PreloadDataset.build(instance.class.to_s) }
+    end
   end
 end

spec/factories/lcpe/preload_datasets.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+FactoryBot.define do
+  factory :lcpe_preload_dataset, class: 'Lcpe::PreloadDataset' do
+    body { 'MyText' }
+    subject_class { 'MyString' }
+  end
+end

spec/models/lcpe/exam_spec.rb

@@ -6,6 +6,7 @@
   let(:version) { create :version, :production }
   let(:institution) { create :institution, version_id: version.id }
   let(:facility_code) { institution.facility_code }
+  let(:preload_id) { Lcpe::PreloadDataset.fresh(described_class.to_s).id }
 
   before { create :weam, facility_code: }
 
@@ -16,22 +17,19 @@
   end
 
   describe '.with_enriched_id' do
-    before { create :lcpe_exam, facility_code: }
+    before { create :lcpe_exam, :preloaded, facility_code: }
 
     it 'returns exams with ref_code and enriched_id attribute' do
       exam_enriched = described_class.with_enriched_id.first
-      ref = generate_ref_code_from(exam_enriched)
-      id = exam_enriched.id.to_s + '@' + ref
-      expect(exam_enriched.ref_code).to eq(ref)
+      id = exam_enriched.id.to_s + '@' + preload_id.to_s
       expect(exam_enriched.enriched_id).to eq(id)
     end
   end
 
   describe '.by_enriched_id' do
-    subject(:exam) { create :lcpe_exam, facility_code: }
+    subject(:exam) { create :lcpe_exam, :preloaded, facility_code: }
 
-    let(:ref_code) { generate_ref_code_from(exam) }
-    let(:enriched_id) { exam.id.to_s + '@' + ref_code }
+    let(:enriched_id) { exam.id.to_s + '@' + preload_id.to_s }
 
     it 'finds Lcpe::Lac by enriched_id' do
       expect(described_class.by_enriched_id(enriched_id).first).to eq(exam)
@@ -44,9 +42,4 @@
       expect(sql).to be_a Lcpe::SqlContext::Sql
     end
   end
-
-  def generate_ref_code_from(exam)
-    hash = exam.facility_code + '-' + exam.nexam_nm
-    Digest::MD5.hexdigest(hash).last(5)
-  end
 end

spec/models/lcpe/lac_spec.rb

@@ -6,6 +6,7 @@
   let(:version) { create :version, :production }
   let(:institution) { create :institution, version_id: version.id }
   let(:facility_code) { institution.facility_code }
+  let(:preload_id) { Lcpe::PreloadDataset.fresh(described_class.to_s).id }
 
   before { create :weam, facility_code: }
 
@@ -16,22 +17,19 @@
   end
 
   describe '.with_enriched_id' do
-    before { create :lcpe_lac, facility_code: }
+    before { create :lcpe_lac, :preloaded, facility_code: }
 
     it 'returns lacs with ref_code and enriched_id attribute' do
       lac_enriched = described_class.with_enriched_id.first
-      ref = generate_ref_code_from(lac_enriched)
-      id = lac_enriched.id.to_s + '@' + ref
-      expect(lac_enriched.ref_code).to eq(ref)
+      id = lac_enriched.id.to_s + '@' + preload_id.to_s
       expect(lac_enriched.enriched_id).to eq(id)
     end
   end
 
   describe '.by_enriched_id' do
-    subject(:lac) { create :lcpe_lac, facility_code: }
+    subject(:lac) { create :lcpe_lac, :preloaded, facility_code: }
 
-    let(:ref_code) { generate_ref_code_from(lac) }
-    let(:enriched_id) { lac.id.to_s + '@' + ref_code }
+    let(:enriched_id) { lac.id.to_s + '@' + preload_id.to_s }
 
     it 'finds Lcpe::Lac by enriched_id' do
       expect(described_class.by_enriched_id(enriched_id).first).to eq(lac)
@@ -44,9 +42,4 @@
       expect(sql).to be_a Lcpe::SqlContext::Sql
     end
   end
-
-  def generate_ref_code_from(lac)
-    hash = lac.facility_code + '-' + lac.lac_nm
-    Digest::MD5.hexdigest(hash).last(5)
-  end
 end