Add Elasticsearch/OpenSearch support

.github/matrix.json

@@ -50,6 +50,14 @@
       "rails_version": "7.2.3",
       "additional_engine_cart_rails_options": "-a propshaft --css=bootstrap --js=esbuild",
       "additional_name": "| Rails 7.2 + Propshaft, esbuild"
+    },
+    {
+      "ruby": "3.4",
+      "rails_version": "8.1.2",
+      "search_engine": "elasticsearch",
+      "experimental": true,
+      "additional_engine_cart_rails_options": "--css=bootstrap",
+      "additional_name": "| Elasticsearch"
     }
   ]
 }

.github/workflows/test.yml

@@ -26,6 +26,9 @@ jobs:
     needs: set_matrix
     runs-on: ubuntu-latest
     name: ruby ${{ matrix.ruby }} | rails ${{ matrix.rails_version }} ${{ matrix.additional_name }}
+    # The Elasticsearch configuration is still reaching feature parity with Solr,
+    # so it is allowed to fail without failing the overall build.
+    continue-on-error: ${{ matrix.experimental == true }}
     strategy:
       fail-fast: false
       matrix: ${{fromJson(needs.set_matrix.outputs.matrix)}}
@@ -35,6 +38,8 @@ jobs:
       VIEW_COMPONENT_VERSION: ${{ matrix.view_component_version }}
       BOOTSTRAP_VERSION: ${{ matrix.bootstrap_version }}
       BLACKLIGHT_API_TEST: ${{ matrix.api }}
+      BLACKLIGHT_ADAPTER: ${{ matrix.search_engine }}
+      ELASTICSEARCH_URL: http://localhost:9200
       ENGINE_CART_RAILS_OPTIONS: "--skip-git --skip-listen --skip-spring --skip-keeps --skip-kamal --skip-thruster --skip-solid --skip-coffee --skip-test ${{ matrix.additional_engine_cart_rails_options }}"
       BUNDLER_VERSION: ${{ matrix.bundler_version || '2.7.2' }}
     steps:

README.md

@@ -60,6 +60,45 @@ Code contributions are always welcome, instructions for contributing can be foun
 ## Configuring Apache Solr
 You'll also want some information about how Blacklight expects [Apache Solr](http://lucene.apache.org/solr ) to run, which you can find in [Solr Configuration](https://github.com/projectblacklight/blacklight/wiki/Solr-Configuration#solr-configuration)
 
+## Using Elasticsearch / OpenSearch (experimental)
+Blacklight defaults to Apache Solr, but it can also run against an Elasticsearch
+(or API-compatible OpenSearch) cluster. Select the adapter in
+`config/blacklight.yml`:
+
+```yaml
+development:
+  adapter: elasticsearch
+  url: http://127.0.0.1:9200
+  index: blacklight-core
+```
+
+Applications using this adapter must add an Elasticsearch client to their
+`Gemfile` (the gem is not a runtime dependency of Blacklight):
+
+```ruby
+bundle add elasticsearch
+# or, for OpenSearch:
+# bundle add opensearch-ruby
+```
+
+With the adapter selected, the generated `SearchBuilder` and `SolrDocument`
+classes automatically mix in the correct behavior (`include
+Blacklight.search_builder_behavior` / `include Blacklight.document_mixin`).
+
+Some features that depend on Solr-specific functionality are not available when
+using Elasticsearch and are automatically disabled: spellcheck/"did you mean",
+result grouping, pivot and query facets, more-like-this, autocomplete
+suggestions, and the Solr JSON Query DSL advanced search.
+
+`rails blacklight:index:seed` will create the index (if needed) and load the
+sample data. The default index mapping understands Blacklight's Solr field
+naming conventions: fields ending in a text suffix (e.g. `title_tsim`) are
+mapped as analyzed `text` for full-text search, while other string fields
+(e.g. `format`, `language_ssim`, `pub_date_si`) are mapped as `keyword` for
+filtering, sorting, and faceting. Override the mapping with
+`config.elasticsearch_index_settings` in your `CatalogController` if you need
+full control over the schema.
+
 ## Building the javascript
 The javascript includes some derivative combination files that are built at release time, that can be used by some javascript pipelines. The derivatives are placed at `app/assets/javascripts/blacklight`, and files there should not be edited by hand.
 

app/services/blacklight/bookmarks_search_builder.rb

@@ -6,16 +6,15 @@ class BookmarksSearchBuilder < ::SearchBuilder
     ##
     # Filters the query to only include the bookmarked items
     #
-    # @param [Hash] solr_parameters
+    # @param [Blacklight::Solr::Request, Blacklight::ElasticSearch::Request] request_parameters
     #
     # @return [void]
-    def bookmarked(solr_parameters)
-      solr_parameters[:fq] ||= []
+    def bookmarked(request_parameters)
       bookmarks = @scope.context.fetch(:bookmarks)
       return unless bookmarks
 
       document_ids = bookmarks.collect { |b| b.document_id.to_s }
-      solr_parameters[:fq] += ["{!terms f=id}#{document_ids.join(',')}"]
+      request_parameters.append_ids_filter(blacklight_config.document_model.unique_key, document_ids)
     end
     self.default_processor_chain += [:bookmarked]
   end

app/services/blacklight/search_service.rb

@@ -126,36 +126,43 @@ def solr_opensearch_params(field)
     # Pagination parameters for selecting the previous and next documents
     # out of a result set.
     def previous_and_next_document_params(index, window = 1)
-      solr_params = blacklight_config.document_pagination_params.dup
+      pagination_params = blacklight_config.document_pagination_params.dup
 
-      if solr_params.empty?
-        solr_params[:fl] = blacklight_config.document_model.unique_key
-      end
+      # Fill in the repository-appropriate defaults (e.g. Solr `fl`/`facet` or
+      # Elasticsearch `_source`) for fetching a minimal, unfaceted result set.
+      pagination_params.reverse_merge!(repository.default_document_pagination_params(blacklight_config.document_model.unique_key))
 
       if index > 0
-        solr_params[:start] = index - window # get one before
-        solr_params[:rows] = (2 * window) + 1 # and one after
+        pagination_params[:start] = index - window # get one before
+        pagination_params[:rows] = (2 * window) + 1 # and one after
       else
-        solr_params[:start] = 0 # there is no previous doc
-        solr_params[:rows] = 2 * window # but there should be one after
+        pagination_params[:start] = 0 # there is no previous doc
+        pagination_params[:rows] = 2 * window # but there should be one after
       end
 
-      solr_params[:facet] = false
-      solr_params
+      pagination_params
     end
 
     ##
     # Retrieve a set of documents by id
     # @param [Array] ids
     # @param [HashWithIndifferentAccess] extra_controller_params
     def fetch_many(ids, extra_controller_params)
-      extra_controller_params ||= {}
+      extra_controller_params = (extra_controller_params || {}).dup
+
+      # `rows` is Blacklight's pagination vocabulary (callers such as the bookmarks
+      # controller pass it to fetch all the requested documents rather than the
+      # default page size). Route it through the builder so each adapter translates
+      # it appropriately (Solr `rows`, Elasticsearch `size`) instead of leaking raw
+      # into the request body.
+      rows = extra_controller_params.delete(:rows) || extra_controller_params.delete('rows')
 
       query = search_builder
               .with(search_state)
               .where(blacklight_config.document_model.unique_key => ids)
               .merge(blacklight_config.fetch_many_document_params)
               .merge(extra_controller_params)
+      query.rows(rows) if rows
 
       # find_many was introduced in Blacklight 8.4. Before that, we used the
       # regular search method (possibly with a find-many specific `qt` parameter).

blacklight.gemspec

@@ -35,6 +35,7 @@ Gem::Specification.new do |s|
   s.add_dependency "zeitwerk"
 
   s.add_development_dependency "rsolr", ">= 1.0.6", "< 3"  # Library for interacting with rSolr.
+  s.add_development_dependency "elasticsearch", ">= 8.0", "< 10"  # Client for the optional Elasticsearch/OpenSearch adapter (client major version should match the server).
   s.add_development_dependency "rspec-rails", "~> 7.0"
   s.add_development_dependency "rspec-collection_matchers", ">= 1.0"
   s.add_development_dependency 'axe-core-rspec'

compose.yaml

@@ -36,3 +36,23 @@ services:
       - /opt/solr/conf
       - "-Xms256m"
       - "-Xmx512m"
+
+  # Optional search engine for running Blacklight against Elasticsearch/OpenSearch
+  # instead of Solr. Enable by setting BLACKLIGHT_ADAPTER=elasticsearch.
+  elasticsearch:
+    image: "elasticsearch:${ELASTICSEARCH_VERSION:-9.0.4}"
+    environment:
+      - discovery.type=single-node
+      - xpack.security.enabled=false
+      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+    ports:
+      - "${ELASTICSEARCH_PORT:-9200}:9200"
+    healthcheck:
+      test:
+        [
+          "CMD-SHELL",
+          "curl -fsS http://localhost:9200/_cluster/health || exit 1",
+        ]
+      interval: 10s
+      timeout: 5s
+      retries: 12

lib/blacklight.rb

@@ -36,13 +36,45 @@ def self.repository_class
     case connection_config&.fetch(:adapter) || 'solr'
     when 'solr'
       Blacklight::Solr::Repository
+    when 'elasticsearch', 'elastic_search', 'opensearch'
+      Blacklight::ElasticSearch::Repository
     when /::/
       connection_config[:adapter].constantize
     else
       Blacklight.const_get("#{connection_config.fetch(:adapter)}/Repository".classify)
     end
   end
 
+  ##
+  # The response model class appropriate for the configured adapter.
+  # @return [Class]
+  def self.default_response_model
+    repository_class.try(:response_model) || Blacklight::Solr::Response
+  end
+
+  ##
+  # The facet paginator class appropriate for the configured adapter.
+  # @return [Class]
+  def self.default_facet_paginator_class
+    repository_class.try(:facet_paginator_class) || Blacklight::Solr::FacetPaginator
+  end
+
+  ##
+  # The SearchBuilder behavior module appropriate for the configured adapter.
+  # Intended to be included into an application's SearchBuilder.
+  # @return [Module]
+  def self.search_builder_behavior
+    repository_class.try(:search_builder_behavior) || Blacklight::Solr::SearchBuilderBehavior
+  end
+
+  ##
+  # The document mixin appropriate for the configured adapter.
+  # Intended to be included into an application's document model.
+  # @return [Module]
+  def self.document_mixin
+    repository_class.try(:document_mixin) || Blacklight::Solr::Document
+  end
+
   ##
   # The default Blacklight configuration.
   def self.default_configuration

lib/blacklight/abstract_repository.rb

@@ -54,6 +54,19 @@ def reflect_fields
       raise NotImplementedError
     end
 
+    # Repository-specific request parameters for fetching documents with a
+    # minimal field set and without faceting. Used by
+    # Blacklight::SearchService when paging to the previous/next document.
+    #
+    # The default is the Solr-style parameters (preserved for Solr and any
+    # custom adapters); other adapters (e.g. Elasticsearch) override this.
+    #
+    # @param [String] unique_key the document model's unique key field
+    # @return [Hash]
+    def default_document_pagination_params(unique_key)
+      { fl: unique_key, facet: false }
+    end
+
     ##
     # Is the repository in a working state?
     def ping

lib/blacklight/configuration.rb

@@ -106,7 +106,7 @@ def initialized_default_configuration?
       # @!attribute response_model
       # model that maps index responses to the blacklight response model
       # @return [Class]
-      property :response_model, default: Blacklight::Solr::Response
+      property :response_model, default: Blacklight.default_response_model
       # @!attribute document_model
       # the model to use for each response document
       # @return [Class]
@@ -118,7 +118,30 @@ def initialized_default_configuration?
       # @!attribute facet_paginator_class
       # Class for paginating long lists of facet fields
       # @return [Class]
-      property :facet_paginator_class, default: Blacklight::Solr::FacetPaginator
+      property :facet_paginator_class, default: Blacklight.default_facet_paginator_class
+
+      # @!attribute elasticsearch_index
+      # @since v9.0.0
+      # @return [String, nil] the Elasticsearch index (or alias) to search against.
+      #   Only used by the Elasticsearch adapter; may also be set via the `index`
+      #   key in blacklight.yml.
+      property :elasticsearch_index, default: nil
+      # @!attribute elasticsearch_query_fields
+      # @since v9.0.0
+      # @return [Array<String>, nil] the fields a full-text query should target.
+      #   When nil, a simple_query_string across all fields is used.
+      property :elasticsearch_query_fields, default: nil
+      # @!attribute elasticsearch_source_fields
+      # @since v9.0.0
+      # @return [Array<String>, nil] restrict the Elasticsearch `_source` fields
+      #   returned for each document. When nil, the full source is returned.
+      property :elasticsearch_source_fields, default: nil
+      # @!attribute elasticsearch_index_settings
+      # @since v9.0.0
+      # @return [Hash, nil] the body (settings + mappings) used when the
+      #   Elasticsearch adapter creates the index. When nil, a default mapping
+      #   based on Blacklight's Solr field-naming conventions is used.
+      property :elasticsearch_index_settings, default: nil
       # @!attribute connection_config
       # repository connection configuration
       # @since v5.13.0

lib/blacklight/elastic_search.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Blacklight
+  # Adapter for using an Elasticsearch (or API-compatible OpenSearch) cluster as
+  # the search index backing a Blacklight application.
+  #
+  # The adapter is selected by setting `adapter: elasticsearch` in
+  # `config/blacklight.yml`. Some Solr-specific features (spellcheck/"did you
+  # mean", result grouping, pivot/query facets, more-like-this, and the Solr
+  # JSON Query DSL advanced search) are not provided by Elasticsearch and are
+  # automatically disabled when this adapter is in use.
+  module ElasticSearch
+  end
+end

lib/blacklight/elastic_search/document.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Blacklight::ElasticSearch
+  # Mixin for a class representing a single document returned from
+  # Elasticsearch. This is the Elasticsearch analog to
+  # Blacklight::Solr::Document.
+  module Document
+    extend ActiveSupport::Concern
+    include Blacklight::Document
+    include Blacklight::Document::ActiveModelShim
+
+    # More-like-this is not supported by the Elasticsearch adapter.
+    def more_like_this
+      []
+    end
+
+    def has_highlight_field?(field)
+      highlighting = self['_highlighting']
+      return false if highlighting.blank?
+
+      highlighting.key?(field.to_s)
+    end
+
+    def highlight_field(field)
+      return unless has_highlight_field?(field)
+
+      Array(self['_highlighting'][field.to_s]).map(&:html_safe)
+    end
+  end
+end

lib/blacklight/elastic_search/facet_paginator.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module Blacklight::ElasticSearch
+  # Pagination for facet values returned by Elasticsearch terms aggregations.
+  #
+  # Like Solr, Elasticsearch terms aggregations don't return a total count of
+  # distinct values, so we request `limit + 1` values to detect whether more
+  # values are available. We subclass the Solr paginator so that adapter-agnostic
+  # callers (and specs) that check for `Blacklight::Solr::FacetPaginator` work.
+  class FacetPaginator < Blacklight::Solr::FacetPaginator
+  end
+end