Merge pull request #185 from MITLibraries/tco-122-associate-suggested-resource-and-term

Refactor SuggestedResource to leverage new Fingerprint model

Author: jazairi

README.md

@@ -47,9 +47,13 @@ config:
   webroot: .
 ```
 
+We use Lando here because its use in our WordPress environment. However, any static local webserver will work.
+
 If you need to regenerate these cassettes, the following procedure should be sufficient:
 
-1. Use the configuration above to ensure the needed files are visible at `http://static.lndo.site/filename.ext`.
+1. Use the configuration above to ensure the needed files are visible at `http://static.lndo.site/filename.ext` (i.e.,
+run `lando start` in `tacos/test/fixtures/files`). If you are using a server other than Lando, configure it such that
+`tacos/test/fixtures/files` is the root directory, then start the server.
 2. Delete any existing cassette files which need to be regenerated.
 3. Run the test(s).
 4. Commit the resulting files along with your other work.

app/dashboards/detector/suggested_resource_dashboard.rb

@@ -1,93 +1,10 @@
 # frozen_string_literal: true
 
-# == Schema Information
-#
-# Table name: detector_suggested_resources
-#
-#  id          :integer          not null, primary key
-#  title       :string
-#  url         :string
-#  phrase      :string
-#  fingerprint :string
-#  created_at  :datetime         not null
-#  updated_at  :datetime         not null
-#
-
 require 'stringex/core_ext'
 
 class Detector
-  # Detector::SuggestedResource stores custom hints that we want to send to the
-  # user in response to specific strings. For example, a search for "web of
-  # science" should be met with our custom login link to Web of Science via MIT.
-  class SuggestedResource < ApplicationRecord
-    before_save :update_fingerprint
-
-    def self.table_name_prefix
-      'detector_'
-    end
-
-    # This exists for the before_save lifecycle hook to call the calculate_fingerprint method, to ensure that these
-    # records always have a correctly-calculated fingerprint. It has no arguments and returns nothing.
-    def update_fingerprint
-      self.fingerprint = Detector::SuggestedResource.calculate_fingerprint(phrase)
-    end
-
-    # This implements the OpenRefine fingerprinting algorithm. See
-    # https://openrefine.org/docs/technical-reference/clustering-in-depth#fingerprint
-    #
-    # @param old_phrase [String] A text string which needs to have its fingerprint calculated. This could either be the
-    #   "phrase" field on the SuggestedResource record, or an incoming search term received from a contributing system.
-    #
-    # @return [String] A string of all words in the input, downcased, normalized, and alphabetized.
-    def self.calculate_fingerprint(old_phrase)
-      modified_phrase = old_phrase
-      modified_phrase = modified_phrase.strip
-      modified_phrase = modified_phrase.downcase
-
-      # This removes all punctuation and symbol characters from the string.
-      modified_phrase = modified_phrase.gsub(/\p{P}|\p{S}/, '')
-
-      # Normalize to ASCII (e.g. gödel and godel are liable to be intended to
-      # find the same thing)
-      modified_phrase = modified_phrase.to_ascii
-
-      # Coercion to ASCII can introduce new symbols, so we remove those now.
-      modified_phrase = modified_phrase.gsub(/\p{P}|\p{S}/, '')
-
-      # Tokenize
-      tokens = modified_phrase.split
-
-      # Remove duplicates and sort
-      tokens = tokens.uniq
-      tokens = tokens.sort
-
-      # Rejoin tokens
-      tokens.join(' ')
-    end
-
-    # This replaces all current Detector::SuggestedResource records with a new set from an imported CSV.
-    #
-    # @note This method is called by the suggested_resource:reload rake task.
-    #
-    # @param input [CSV::Table] An imported CSV file containing all Suggested Resource records. The CSV file must have
-    #                           at least three headers, named "Title", "URL", and "Phrase". Please note: these values
-    #                           are case sensitive.
-    def self.bulk_replace(input)
-      raise ArgumentError.new, 'Tabular CSV is required' unless input.instance_of?(CSV::Table)
-
-      # Need to check what columns exist in input
-      required_headers = %w[Title URL Phrase]
-      missing_headers = required_headers - input.headers
-      raise ArgumentError.new, "Some CSV columns missing: #{missing_headers}" unless missing_headers.empty?
-
-      Detector::SuggestedResource.delete_all
-
-      input.each do |line|
-        record = Detector::SuggestedResource.new({ title: line['Title'], url: line['URL'], phrase: line['Phrase'] })
-        record.save
-      end
-    end
-
+  # Detector::SuggestedResource handles detections for SuggestedResource records.
+  class SuggestedResource
     # Identify any SuggestedResource record whose pre-calculated fingerprint matches the fingerprint of the incoming
     # phrase.
     #
@@ -98,7 +15,7 @@ def self.bulk_replace(input)
     #
     # @return [Detector::SuggestedResource] The record whose fingerprint matches that of the search term.
     def self.full_term_match(phrase)
-      SuggestedResource.where(fingerprint: calculate_fingerprint(phrase))
+      ::SuggestedResource.joins(:fingerprints).where(fingerprints: { value: Fingerprint.calculate(phrase) })
     end
 
     # Look up any matching Detector::SuggestedResource records, building on the full_term_match method. If a match is

app/models/detector/suggested_resource.rb

@@ -17,7 +17,6 @@
 #  suggested_resource_exact :integer
 #  lcsh                     :integer
 #  citation                 :integer
-#  barcode                  :integer
 #
 module Metrics
   # Algorithms aggregates statistics for matches for all SearchEvents

app/models/metrics/algorithms.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+# == Schema Information
+#
+# Table name: suggested_resources
+#
+#  id         :integer          not null, primary key
+#  title      :string
+#  url        :string
+#  created_at :datetime         not null
+#  updated_at :datetime         not null
+#
+# SuggestedResource stores custom hints that we want to send to the
+# user in response to specific strings. For example, a search for "web of
+# science" should be met with our custom login link to Web of Science via MIT.
+class SuggestedResource < ApplicationRecord
+  has_many :terms, dependent: :nullify
+  has_many :fingerprints, through: :terms, dependent: :nullify
+
+  # This replaces all current SuggestedResource records with a new set from an imported CSV.
+  #
+  # @note This method is called by the suggested_resource:reload rake task.
+  #
+  # @param input [CSV::Table] An imported CSV file containing all Suggested Resource records. The CSV file must have
+  #                           at least three headers, named "Title", "URL", and "Phrase". Please note: these values
+  #                           are case sensitive.
+  def self.bulk_replace(input)
+    raise ArgumentError.new, 'Tabular CSV is required' unless input.instance_of?(CSV::Table)
+
+    # Need to check what columns exist in input
+    required_headers = %w[title url phrase]
+    missing_headers = required_headers - input.headers
+    raise ArgumentError.new, "Some CSV columns missing: #{missing_headers}" unless missing_headers.empty?
+
+    SuggestedResource.destroy_all
+
+    input.each do |line|
+      term = Term.find_or_create_by(phrase: line['phrase'])
+
+      # check for existing SuggestedResource with the same title/url
+      dup_check = SuggestedResource.where(title: line['title'], url: line['url'])
+
+      # link to existing SuggestedResource if one exists
+      term.suggested_resource = if dup_check.count.positive?
+                                  dup_check.first
+                                # create a new SuggestedResource if it doesn't exist
+                                else
+                                  SuggestedResource.new({ title: line['title'], url: line['url'] })
+                                end
+      term.save
+    end
+  end
+end

app/models/suggested_resource.rb

@@ -7,21 +7,24 @@
 #
 # Table name: terms
 #
-#  id             :integer          not null, primary key
-#  phrase         :string
-#  created_at     :datetime         not null
-#  updated_at     :datetime         not null
-#  flag           :boolean
-#  fingerprint_id :integer
+#  id                    :integer          not null, primary key
+#  phrase                :string
+#  created_at            :datetime         not null
+#  updated_at            :datetime         not null
+#  flag                  :boolean
+#  fingerprint_id        :integer
+#  suggested_resource_id :integer
 #
 class Term < ApplicationRecord
   has_many :search_events, dependent: :destroy
   has_many :detections, dependent: :destroy
   has_many :categorizations, dependent: :destroy
   has_many :confirmations, dependent: :destroy
   belongs_to :fingerprint, optional: true
+  belongs_to :suggested_resource, optional: true
 
   before_save :register_fingerprint
+  before_destroy :check_suggested_resource
   after_destroy :check_fingerprint_count
 
   scope :categorized, -> { where.associated(:categorizations).distinct }
@@ -104,6 +107,17 @@ def check_fingerprint_count
     fingerprint.destroy if fingerprint&.terms&.count&.zero?
   end
 
+  # This is called before_destroy to avoid orphaning SuggestedResource records. Deleting terms should be an unlikely
+  # event, so this should come up rarely. If it does, it warrants the extra care to delete the record manually in the
+  # Rails console.
+  def check_suggested_resource
+    return unless suggested_resource
+
+    Rails.logger.error('Cannot delete term with associated suggested resource')
+    Sentry.capture_message('Cannot delete term with associated suggested resource')
+    throw :abort
+  end
+
   # This method looks up all current detections for the given term, and assembles their confidence scores in a format
   # usable by the calculate_categorizations method. It exists to transform data like:
   # [{3=>0.91}, {1=>0.1}] and [{3=>0.95}]

app/models/term.rb

@@ -19,9 +19,6 @@
             <% if can? :view, :playground %>
               <%= link_to('Playground', '/playground', class: 'nav-item') %>
             <% end %>
-            <% if can? :manage, :detector__suggested_resource %>
-              <%= link_to('Suggested Resources', admin_detector_suggested_resources_path, class: 'nav-item') %>
-            <% end %>
             <% if can? :view, Categorization %>
               <%= link_to('Categorizations', admin_categorizations_path, class: 'nav-item') %>
             <% end %>

app/views/layouts/_site_nav.html.erb

@@ -76,6 +76,10 @@
   # Raise error when a before_action's only/except options reference missing actions.
   config.action_controller.raise_on_missing_callback_actions = true
 
+  # Local logging overrides
+  config.logger = Logger.new(STDOUT)
+  config.log_level = :debug
+
   # Apply autocorrection by RuboCop to files generated by `bin/rails generate`.
   # config.generators.apply_rubocop_autocorrect_after_generate!
 end

config/environments/development.rb

@@ -5,11 +5,6 @@
   end
 
   namespace :admin do
-    # Lookup-style detector records
-    namespace :detector do
-      resources :suggested_resources
-    end
-
     # Knowledge graph models
     resources :detectors
     resources :detector_categories

config/routes.rb

@@ -0,0 +1,10 @@
+class CreateSuggestedResources < ActiveRecord::Migration[7.1]
+  def change
+    create_table :suggested_resources do |t|
+      t.string :title
+      t.string :url
+
+      t.timestamps
+    end
+  end
+end

db/migrate/20250213204800_create_suggested_resources.rb

@@ -0,0 +1,18 @@
+class DropDetectorSuggestedResources < ActiveRecord::Migration[7.1]
+  def up
+    drop_table :detector_suggested_resources
+  end
+
+  def down
+    create_table :detector_suggested_resources do |t|
+      t.string :title
+      t.string :url
+      t.string :phrase
+      t.string :fingerprint
+
+      t.timestamps
+    end
+    add_index :detector_suggested_resources, :phrase, unique: true
+    add_index :detector_suggested_resources, :fingerprint, unique: true
+  end
+end

db/migrate/20250213204813_drop_detector_suggested_resources.rb

@@ -0,0 +1,6 @@
+class AddSuggestedResourceToTerms < ActiveRecord::Migration[7.1]
+  def change
+    add_reference :terms, :suggested_resource
+    add_foreign_key :terms, :suggested_resources, on_delete: :nullify
+  end
+end