Optimize DOI event performance with bulk-preloading

app/controllers/datacite_dois_controller.rb

@@ -460,6 +460,9 @@ def show
           publisher: params[:publisher],
           include_other_registration_agencies: params[:include_other_registration_agencies],
         }
+        # Preload events since we are going to show details
+        # This optimizes the serializer which accesses part_events, citation_events, etc.
+        EventsPreloader.new([doi]).preload!
 
         render(
           json: DataciteDoiSerializer.new(doi, options).serializable_hash.to_json,

app/models/concerns/preloaded_event_relation.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+# Wrapper class to make preloaded event arrays compatible with ActiveRecord::Relation API
+# This allows existing code that calls methods like `pluck`, `map`, `select` to work
+# with in-memory arrays without modification.
+class PreloadedEventRelation
+  include Enumerable
+
+  def initialize(events)
+    @events = Array(events)
+  end
+
+  # Delegate Enumerable methods to the underlying array
+  def each(&block)
+    @events.each(&block)
+  end
+
+  # Delegate common Enumerable methods explicitly
+  def first
+    @events.first
+  end
+
+  def last
+    @events.last
+  end
+
+  def count
+    @events.count
+  end
+
+  # Implement pluck to match ActiveRecord::Relation#pluck behavior
+  # Supports single or multiple column names
+  def pluck(*column_names)
+    if column_names.length == 1
+      column_name = column_names.first
+      @events.map { |event| event.public_send(column_name) }
+    else
+      @events.map { |event| column_names.map { |col| event.public_send(col) } }
+    end
+  end
+
+  # Delegate map to the underlying array
+  def map(&block)
+    @events.map(&block)
+  end
+
+  # Delegate select to the underlying array
+  def select(&block)
+    PreloadedEventRelation.new(@events.select(&block))
+  end
+
+  # Delegate other common Enumerable methods
+  def compact
+    PreloadedEventRelation.new(@events.compact)
+  end
+
+  def uniq
+    PreloadedEventRelation.new(@events.uniq)
+  end
+
+  def sort_by(&block)
+    PreloadedEventRelation.new(@events.sort_by(&block))
+  end
+
+  def group_by(&block)
+    @events.group_by(&block)
+  end
+
+  def inject(*args, &block)
+    @events.inject(*args, &block)
+  end
+
+  def length
+    @events.length
+  end
+
+  def empty?
+    @events.empty?
+  end
+
+  def present?
+    @events.present?
+  end
+
+  def blank?
+    @events.blank?
+  end
+
+  # Allow direct access to the underlying array
+  def to_a
+    @events
+  end
+
+  def to_ary
+    @events
+  end
+end

app/models/datacite_doi.rb

@@ -144,19 +144,14 @@ def self.import_in_bulk(ids, options = {})
 
     # get database records from array of database ids
     selected_dois = DataciteDoi.where(id: ids, type: "DataciteDoi").includes(
-      :client,
+      { client: :provider },
       :media,
-      :view_events,
-      :download_events,
-      :citation_events,
-      :reference_events,
-      :part_events,
-      :part_of_events,
-      :version_events,
-      :version_of_events,
       :metadata
     )
     selected_dois.find_in_batches(batch_size: batch_size) do |dois|
+      # Preload all events for this batch in a single query
+      EventsPreloader.new(dois).preload!
+
       bulk_body = dois.map do |doi|
         {
           index: {

app/models/doi.rb

@@ -2,6 +2,7 @@
 
 require "maremma"
 require "benchmark"
+require_relative "concerns/preloaded_event_relation"
 
 class Doi < ApplicationRecord
   self.ignored_columns += [:publisher]
@@ -76,6 +77,7 @@ class Doi < ApplicationRecord
   alias_attribute :state, :aasm_state
 
   attr_accessor :current_user
+  attr_accessor :preloaded_events
 
   attribute :regenerate, :boolean, default: false
   attribute :only_validate, :boolean, default: false
@@ -100,6 +102,73 @@ class Doi < ApplicationRecord
   # has_many :source_events, class_name: "Event", primary_key: :doi, foreign_key: :source_doi, dependent: :destroy
   # has_many :target_events, class_name: "Event", primary_key: :doi, foreign_key: :target_doi, dependent: :destroy
 
+  # Override association methods to use preloaded_events when available
+  # Check for !nil instead of present? to handle empty arrays (preloaded but no events)
+  # Also filter by source_doi/target_doi to match association behavior
+  def view_events
+    if !preloaded_events.nil?
+      filtered_preloaded_events(:target_relation_type_id, "views", :target_doi)
+    else
+      association(:view_events).scope
+    end
+  end
+
+  def download_events
+    if !preloaded_events.nil?
+      filtered_preloaded_events(:target_relation_type_id, "downloads", :target_doi)
+    else
+      association(:download_events).scope
+    end
+  end
+
+  def reference_events
+    if !preloaded_events.nil?
+      filtered_preloaded_events(:source_relation_type_id, "references", :source_doi)
+    else
+      association(:reference_events).scope
+    end
+  end
+
+  def citation_events
+    if !preloaded_events.nil?
+      filtered_preloaded_events(:target_relation_type_id, "citations", :target_doi)
+    else
+      association(:citation_events).scope
+    end
+  end
+
+  def part_events
+    if !preloaded_events.nil?
+      filtered_preloaded_events(:source_relation_type_id, "parts", :source_doi)
+    else
+      association(:part_events).scope
+    end
+  end
+
+  def part_of_events
+    if !preloaded_events.nil?
+      filtered_preloaded_events(:target_relation_type_id, "part_of", :target_doi)
+    else
+      association(:part_of_events).scope
+    end
+  end
+
+  def version_events
+    if !preloaded_events.nil?
+      filtered_preloaded_events(:source_relation_type_id, "versions", :source_doi)
+    else
+      association(:version_events).scope
+    end
+  end
+
+  def version_of_events
+    if !preloaded_events.nil?
+      filtered_preloaded_events(:target_relation_type_id, "version_of", :target_doi)
+    else
+      association(:version_of_events).scope
+    end
+  end
+
   has_many :references, class_name: "Doi", through: :reference_events, source: :doi_for_target
   has_many :citations, class_name: "Doi", through: :citation_events, source: :doi_for_source
   has_many :parts, class_name: "Doi", through: :part_events, source: :doi_for_target
@@ -2781,6 +2850,15 @@ def handle_resource_type(types)
   end
 
   private
+    def filtered_preloaded_events(relation_type_key, relation_type_value, doi_key)
+      PreloadedEventRelation.new(
+        preloaded_events.select do |e|
+          e.public_send(relation_type_key) == relation_type_value &&
+            e.public_send(doi_key)&.upcase == doi.upcase
+        end
+      )
+    end
+
     def update_publisher_from_hash
       symbolized_publisher_hash = publisher_before_type_cast.symbolize_keys
       if !symbolized_publisher_hash.values.all?(nil)

app/models/other_doi.rb

@@ -148,19 +148,14 @@ def self.import_in_bulk(ids, options = {})
 
     # get database records from array of database ids
     selected_dois = OtherDoi.where(id: ids, type: "OtherDoi").includes(
-      :client,
+      { client: :provider },
       :media,
-      :view_events,
-      :download_events,
-      :citation_events,
-      :reference_events,
-      :part_events,
-      :part_of_events,
-      :version_events,
-      :version_of_events,
       :metadata
     )
     selected_dois.find_in_batches(batch_size: batch_size) do |dois|
+      # Preload all events for this batch in a single query
+      EventsPreloader.new(dois).preload!
+
       bulk_body = dois.map do |doi|
         {
           index: {

app/models/reference_repository.rb

@@ -355,6 +355,11 @@ def filter(options)
             "repository_type": options[:repository_type].split(",")
           } }
         end
+        if options[:provider_id].present?
+          retval << { term: {
+            "provider_id": { value: options[:provider_id], case_insensitive: true }
+          } }
+        end
         if options[:is_open] == "true"
           retval << { term: {
             "data_access.type": "open"

app/services/events_preloader.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+# Service class to preload events for a batch of DOIs in a single query
+# This dramatically reduces database queries from N*M (DOIs * Relationship Types) to 1-2 queries total
+class EventsPreloader
+  # Maximum number of DOIs to query at once to avoid database parameter limits
+  CHUNK_SIZE = 1000
+
+  def initialize(dois)
+    @dois = Array(dois)
+    @doi_map = {}
+    @dois.each do |doi|
+      next if doi.doi.blank?
+      @doi_map[doi.doi.upcase] = doi
+      # Initialize preloaded_events array if not already set
+      doi.preloaded_events ||= []
+    end
+  end
+
+  # Preload all events for the batch of DOIs
+  def preload!
+    return if @dois.empty?
+
+    doi_identifiers = @dois.filter_map { |doi| doi.doi&.upcase }.uniq
+    return if doi_identifiers.empty?
+
+    # Fetch events in chunks to avoid database parameter limits
+    all_events = []
+    doi_identifiers.each_slice(CHUNK_SIZE) do |chunk|
+      events = Event.where(
+        "source_doi IN (?) OR target_doi IN (?)",
+        chunk, chunk
+      ).order(updated_at: :desc).to_a
+      all_events.concat(events)
+    end
+    all_events.uniq!
+
+    # Group events by DOI and assign to each Doi object
+    all_events.each do |event|
+      # Add event to source DOI's preloaded_events if it matches
+      if event.source_doi.present?
+        source_doi_obj = @doi_map[event.source_doi.upcase]
+        source_doi_obj.preloaded_events << event if source_doi_obj
+      end
+
+      # Add event to target DOI's preloaded_events if it matches
+      if event.target_doi.present?
+        target_doi_obj = @doi_map[event.target_doi.upcase]
+        if target_doi_obj && target_doi_obj != source_doi_obj
+          target_doi_obj.preloaded_events << event
+        end
+      end
+    end
+
+    # Ensure all DOIs have an array (even if empty)
+    @dois.each do |doi|
+      doi.preloaded_events ||= []
+    end
+
+    self
+  end
+end