Merge pull request #207 from MITLibraries/tco-141-regexable-suggested-resources

Adds Suggested Resource detection based on regex patterns

Author: JPrevost

app/controllers/demo_controller.rb

@@ -23,5 +23,6 @@ def detections
     @detections[:lcsh] = Detector::Lcsh.new(@searchterm).detections
     @detections[:standard_identifiers] = Detector::StandardIdentifiers.new(@searchterm).detections
     @detections[:suggested_resources] = Detector::SuggestedResource.full_term_match(@searchterm)
+    @detections[:suggested_resources_patterns] = Detector::SuggestedResourcePattern.new(@searchterm)
   end
 end

app/graphql/types/detectors_type.rb

@@ -25,8 +25,19 @@ def standard_identifiers
       end
     end
 
+    # Prefer Term based SuggestedResources over Pattern Based Suggested Resources
     def suggested_resources
-      Detector::SuggestedResource.full_term_match(@object).map do |suggested_resource|
+      traditional_suggested_resources.presence || pattern_based_suggested_resources
+    end
+
+    def traditional_suggested_resources
+      Detector::SuggestedResource.full_term_match(@object) do |suggested_resource|
+        { title: suggested_resource.title, url: suggested_resource.url }
+      end
+    end
+
+    def pattern_based_suggested_resources
+      Detector::SuggestedResourcePattern.new(@object).detections do |suggested_resource|
         { title: suggested_resource.title, url: suggested_resource.url }
       end
     end

app/models/detector/bulk_checker.rb

@@ -5,7 +5,7 @@ class Detector
   # singleton class to access it
   # See also: `PatternChecker` for shared instance methods
   module BulkChecker
-    # This method is intended to be used for inspecting detections during development.
+    # check_all_matches is intended to be used for inspecting detections during development.
     # Assumptions include
     #   - the Class including this module implements a `detections` method (either via `attr_reader` or as a method)
     #     that is only populated for Terms in which it has made a detection
@@ -24,14 +24,19 @@ def check_all_matches(output: false)
         matches.push [t.phrase, d.detections]
       end
 
-      if Rails.env.development?
-        Rails.logger.ap matches
-
-        Rails.logger.ap "Total Terms: #{Term.count}"
-        Rails.logger.ap "Total Matches: #{count}"
-      end
+      log_summary(matches) if Rails.env.development?
 
       matches if output
     end
+
+    # log_summary formats and logs information collected in check_all_matches
+    #
+    # @param matches [array]. matches should be an array of [phrase, detections]
+    def log_summary(matches)
+      Rails.logger.info(ap(matches))
+
+      Rails.logger.info "Total Terms  : #{Term.count}"
+      Rails.logger.info "Total Matches: #{matches.count}"
+    end
   end
 end

app/models/detector/standard_identifiers.rb

@@ -6,10 +6,6 @@ class Detector
   class StandardIdentifiers
     attr_reader :detections
 
-    def self.table_name_prefix
-      'detector_'
-    end
-
     # shared instance methods
     include Detector::PatternChecker
 

app/models/detector/suggested_resource_pattern.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+class Detector
+  # Detector::SuggestedResourcePattern handles detections for patterns stored in our SuggestedPattern model
+  class SuggestedResourcePattern
+    attr_reader :detections
+
+    # shared singleton methods
+    extend Detector::BulkChecker
+
+    def initialize(phrase)
+      @detections = {}
+      check_patterns(phrase)
+    end
+
+    # check_patterns loops through all stored patterns from SuggestedPattern model, checks to see if they produce
+    # matches for the incoming `phrase`, and if so creates a Hash with useful data
+    #
+    # @note Not using shared PatternChecker as we want to include additional data in the returned object
+    # @param phrase [String]. A string representation of a searchterm (not an actual Term object)
+    # @return primarily intended to add matches to @detections
+    def check_patterns(phrase)
+      sps = []
+      SuggestedPattern.find_each do |sp|
+        next unless Regexp.new(sp.pattern).match(phrase)
+
+        sps << {
+          shortcode: sp.shortcode,
+          title: sp.title,
+          url: sp.url
+        }
+        @detections = sps
+      end
+    end
+
+    # The record method will consult the set of regex-based detectors that are defined in
+    # SuggestedPattern records. Any matches will be registered as Detection records.
+    #
+    # @note There are multiple patterns within SuggestedPattern records. Each check is capable of generating
+    #       a separate Detection record.
+    #
+    # @return nil
+    def self.record(term)
+      sp = Detector::SuggestedResourcePattern.new(term.phrase)
+
+      sp.detections.each do
+        Detection.find_or_create_by(
+          term:,
+          detector: Detector.where(name: 'SuggestedResourcePattern').first,
+          detector_version: ENV.fetch('DETECTOR_VERSION', 'unset')
+        )
+      end
+
+      nil
+    end
+  end
+end

app/models/metrics/algorithms.rb

@@ -17,6 +17,7 @@
 #  suggested_resource_exact :integer
 #  lcsh                     :integer
 #  citation                 :integer
+#  barcode                  :integer
 #
 module Metrics
   # Algorithms aggregates statistics for matches for all SearchEvents

app/models/suggested_pattern.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+# == Schema Information
+#
+# Table name: suggested_patterns
+#
+#  id         :integer          not null, primary key
+#  title      :string           not null
+#  url        :string           not null
+#  pattern    :string           not null
+#  shortcode  :string           not null
+#  created_at :datetime         not null
+#  updated_at :datetime         not null
+#
+class SuggestedPattern < ApplicationRecord
+  validates :title, presence: true
+  validates :url, presence: true
+  validates :pattern, presence: true, uniqueness: true
+  validates :shortcode, presence: true, uniqueness: true
+end

app/models/term.rb

@@ -55,6 +55,7 @@ def record_detections
     Detector::Journal.record(self)
     Detector::Lcsh.record(self)
     Detector::SuggestedResource.record(self)
+    Detector::SuggestedResourcePattern.record(self)
 
     nil
   end

app/views/demo/view.html.erb

@@ -108,3 +108,15 @@
 <% else %>
   <p>No suggested resoure found.</p>
 <% end %>
+
+<h2>Suggested Resource (Patterns)</h2>
+<% if @detections[:suggested_resources_patterns].detections.present? %>
+  <p>Suggested resoure patterns found.</p>
+  <% @detections[:suggested_resources_patterns].detections.each do |sr| %>
+    <h3><%= sr[:title] %></h3>
+    <p>URL: <%= link_to(sr[:url], sr[:url]) %></p>
+  <% end %>
+
+<% else %>
+  <p>No suggested resoure patterns found.</p>
+<% end %>

db/migrate/20250331153721_add_suggested_patterns.rb

@@ -0,0 +1,15 @@
+class AddSuggestedPatterns < ActiveRecord::Migration[7.2]
+  def change
+    create_table :suggested_patterns do |t|
+      t.string :title, null: false
+      t.string :url, null: false
+      t.string :pattern, null: false
+      t.string :shortcode, null: false
+
+      t.timestamps
+    end
+
+    add_index :suggested_patterns, :pattern, unique: true
+    add_index :suggested_patterns, :shortcode, unique: true
+  end
+end

db/schema.rb

@@ -42,6 +42,7 @@
 Detector.find_or_create_by(name: 'SuggestedResource')
 Detector.find_or_create_by(name: 'Citation')
 Detector.find_or_create_by(name: 'Barcode')
+Detector.find_or_create_by(name: 'SuggestedResourcePattern')
 
 # DetectorCategories
 DetectorCategory.find_or_create_by(
@@ -85,4 +86,36 @@
   confidence: 0.2
 )
 
+# Patterns for Suggested Resources
+SuggestedPattern.find_or_create_by(
+  title: 'Looking for Standards?',
+  url: 'https://libguides.mit.edu/standards',
+  pattern: '(IEC|iec)(\\s)(\\d{5})',
+  shortcode: 'iec'
+)
+SuggestedPattern.find_or_create_by(
+  title: 'Looking for Standards?',
+  url: 'https://libguides.mit.edu/standards',
+  pattern: '(ASCE|asce)(\\s)(\\d)',
+  shortcode: 'asce'
+)
+SuggestedPattern.find_or_create_by(
+  title: 'Looking for Standards?',
+  url: 'https://libguides.mit.edu/standards',
+  pattern: '(IEEE|ieee)\\s+(?:Std\\s+)?([PC]?[0-9]{3,4})',
+  shortcode: 'ieee'
+)
+SuggestedPattern.find_or_create_by(
+  title: 'Looking for Standards?',
+  url: 'https://libguides.mit.edu/standards',
+  pattern: '(ISO|iso)\\s(\\d{1,5})',
+  shortcode: 'iso'
+)
+SuggestedPattern.find_or_create_by(
+  title: 'Looking for Standards?',
+  url: 'https://libguides.mit.edu/standards',
+  pattern: '(ASTM|astm)\\s',
+  shortcode: 'astm'
+)
+
 Rails.logger.info('Seeding DB complete')

db/seeds.rb

@@ -33,3 +33,6 @@ journal:
 
 suggestedresource:
   name: 'SuggestedResource'
+
+suggestedresourcepattern:
+  name: 'SuggestedResourcePattern'

test/fixtures/detectors.yml

@@ -55,3 +55,5 @@ web_of_knowledge:
 nobel_laureate:
   value: 'bawendi moungi'
 
+astm:
+  value: 'astm 1'

test/fixtures/fingerprints.yml

@@ -0,0 +1,18 @@
+# == Schema Information
+#
+# Table name: suggested_patterns
+#
+#  id         :integer          not null, primary key
+#  title      :string           not null
+#  url        :string           not null
+#  pattern    :string           not null
+#  shortcode  :string           not null
+#  created_at :datetime         not null
+#  updated_at :datetime         not null
+#
+
+astm:
+  title: Looking for ASTM Standards?
+  url: 'https://example.com/standards'
+  pattern: '(ASTM|astm)\s'
+  shortcode: astm

test/fixtures/suggested_patterns.yml

@@ -83,3 +83,6 @@ nobel_laureate:
   fingerprint: nobel_laureate
   suggested_resource: nobel_laureate
 
+astm:
+  phrase: 'astm 1'
+  fingerprint: astm

test/fixtures/terms.yml

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require 'test_helper'
+
+class Detector
+  class SuggestedResourcePatternTest < ActiveSupport::TestCase
+    test 'pattern matches return as expected' do
+      match = SuggestedResourcePattern.new('astm standard thing and stuff')
+
+      assert_predicate(match.detections, :present?)
+    end
+
+    test 'no patterns detected return as expected' do
+      match = SuggestedResourcePattern.new('hello!')
+
+      assert_not_predicate(match.detections, :present?)
+    end
+
+    test 'record does relevant work' do
+      detection_count = Detection.count
+      t = terms('astm')
+      Detector::SuggestedResourcePattern.record(t)
+
+      assert_equal(detection_count + 1, Detection.count)
+    end
+
+    test 'record does nothing when not needed' do
+      detection_count = Detection.count
+      t = terms('journal_nature_medicine')
+
+      Detector::SuggestedResourcePattern.record(t)
+
+      assert_equal(detection_count, Detection.count)
+    end
+
+    test 'record respects changes to the DETECTOR_VERSION value' do
+      # Create a relevant detection
+      t = terms('astm')
+      Detector::SuggestedResourcePattern.record(t)
+
+      detection_count = Detection.count
+
+      # Calling the record method again doesn't do anything, but does not error.
+      Detector::SuggestedResourcePattern.record(t)
+
+      assert_equal(detection_count, Detection.count)
+
+      # Calling the record method after DETECTOR_VERSION is incremented results in a new Detection
+      ClimateControl.modify DETECTOR_VERSION: 'updated' do
+        Detector::SuggestedResourcePattern.record(t)
+
+        assert_equal detection_count + 1, Detection.count
+      end
+    end
+  end
+end

test/models/detector/suggested_resource_pattern_test.rb

@@ -1,17 +1,5 @@
 # frozen_string_literal: true
 
-# == Schema Information
-#
-# Table name: suggested_resources
-#
-#  id          :integer          not null, primary key
-#  title       :string
-#  url         :string
-#  phrase      :string
-#  fingerprint :string
-#  created_at  :datetime         not null
-#  updated_at  :datetime         not null
-#
 require 'test_helper'
 
 class Detector

test/models/detector/suggested_resource_test.rb

@@ -17,6 +17,7 @@
 #  suggested_resource_exact :integer
 #  lcsh                     :integer
 #  citation                 :integer
+#  barcode                  :integer
 #
 require 'test_helper'