Merge pull request #572 from alphagov/add-postcode-side-index

Index update, Phase 2: Use new large_user_postcode index

Author: KludgeKML

app/models/postcode.rb

@@ -7,6 +7,9 @@ class Postcode < ApplicationRecord
   scope :active, -> { where(retired: false) }
   scope :retired, -> { where(retired: true) }
 
+  scope :small, -> { where(large_user_postcode: false) }
+  scope :large, -> { where(large_user_postcode: true) }
+
 private
 
   def normalize_postcode

app/workers/ons_import_worker.rb

@@ -15,12 +15,14 @@ def perform(s3_key_name)
       next if Postcode.os_places.where(postcode:).any?
 
       termination_date = parse_termination_date(row["doterm"])
+      large_user_postcode = (row["usertype"] != "0")
+
       results = [
         {
           "ONS" => {
             "AVG_LNG" => row["long"],
             "AVG_LAT" => row["lat"],
-            "TYPE" => row["usertype"] == "0" ? "S" : "L",
+            "TYPE" => large_user_postcode ? "L" : "S",
             "DOTERM" => termination_date,
           },
         },
@@ -30,9 +32,9 @@ def perform(s3_key_name)
 
       existing_record = Postcode.onspd.find_by(postcode:)
       if existing_record
-        existing_record.update(retired:, results:)
+        existing_record.update(retired:, large_user_postcode:, results:)
       else
-        Postcode.create(postcode:, source: "onspd", retired:, results:)
+        Postcode.create(postcode:, source: "onspd", retired:, large_user_postcode:, results:)
       end
     end
   rescue StandardError => e

app/workers/postcodes_collection_worker.rb

@@ -12,9 +12,7 @@ def perform
 
   def postcodes
     Postcode.uncached do
-      onspd_candidates = Postcode.onspd.active.order("updated_at ASC").select { |r| r.results.first["ONS"]["TYPE"] == "S" }.first(POSTCODES_PER_SECOND)
-      os_places_candidates = Postcode.os_places.order("updated_at ASC").first(POSTCODES_PER_SECOND)
-      (onspd_candidates + os_places_candidates).sort_by(&:updated_at).first(POSTCODES_PER_SECOND).pluck(:postcode)
+      Postcode.active.small.order("updated_at ASC").limit(POSTCODES_PER_SECOND).pluck(:postcode)
     end
   end
 end

spec/workers/postcodes_collection_worker_spec.rb

@@ -20,7 +20,7 @@
 
     context "when an ONSPD small active postcode is older than the oldest OS Places postcode" do
       before do
-        Postcode.create(postcode: "E12AA", source: "onspd", updated_at: 3.days.ago, results: [{ "ONS" => { "TYPE" => "S" } }])
+        Postcode.create(postcode: "E12AA", source: "onspd", updated_at: 3.days.ago)
       end
 
       it "adds the onspd postcode to the update checker" do
@@ -35,7 +35,7 @@
 
     context "when an ONSPD small retired postcode is older than the oldest OS Places postcode" do
       before do
-        Postcode.create(postcode: "E12AA", source: "onspd", updated_at: 3.days.ago, results: [{ "ONS" => { "TYPE" => "S" } }], retired: true)
+        Postcode.create(postcode: "E12AA", source: "onspd", updated_at: 3.days.ago, retired: true)
       end
 
       it "does not attempt to update the ONSPD postcode" do
@@ -47,7 +47,7 @@
 
     context "when an ONSPD large active postcode is older than the oldest OS Places postcode" do
       before do
-        Postcode.create(postcode: "E12AA", source: "onspd", updated_at: 3.days.ago, results: [{ "ONS" => { "TYPE" => "L" } }])
+        Postcode.create(postcode: "E12AA", source: "onspd", updated_at: 3.days.ago, large_user_postcode: true)
       end
 
       it "does not attempt to update the ONSPD postcode" do