Use a separate request handler for similarity searches

Register a /select_similarity request handler on the freesound
collection and use it from search_sounds when similar_to is not None

This allows us to check metrics for normal search or similarity search
in grafana

Author: alastair

search/solrapi.py

@@ -200,6 +200,9 @@ def create_collection_and_schema(self, delete_default_fields_definition, schema_
         if copyfields:
             self.create_copyfields(copyfields)
 
+        for handler in schema_definition.get("requestHandlers", []):
+            self.upsert_request_handler(handler["name"], handler["class"], defaults=handler.get("defaults"))
+
     def collection_exists(self):
         if self.logging_verbose >= 1:
             logger.info(f"Checking if collection {self.collection} exists")
@@ -237,6 +240,36 @@ def disable_auto_data_driven_schema(self):
         if self.logging_verbose >= 1:
             logger.info(f"Disabled auto data driven schema for collection {self.collection}")
 
+    def upsert_request_handler(self, name, class_name, defaults=None):
+        """Idempotently add or update a Solr request handler.
+
+        Tries add-requesthandler first. Solr's Config API returns HTTP 200 with
+        an errorMessages payload (rather than 4xx) when a command is rejected —
+        e.g. when the handler already exists — so we detect that and retry with
+        update-requesthandler. No separate GET, so no TOCTOU window between the
+        existence check and the write.
+        """
+        handler_def = {"name": name, "class": class_name}
+        if defaults is not None:
+            handler_def["defaults"] = defaults
+
+        url = urljoin(self.base_url, f"api/collections/{self.collection}/config")
+        body = self._make_post(url, {"add-requesthandler": handler_def})
+        if not body.get("errorMessages"):
+            if self.logging_verbose >= 1:
+                logger.info(f"add-requesthandler {name} on collection {self.collection}")
+            return
+
+        # Add was rejected — most likely because the handler already exists.
+        # Retry as update; if that also reports errorMessages, surface the failure.
+        if self.logging_verbose >= 2:
+            logger.info(f"add-requesthandler {name} rejected by Solr ({body['errorMessages']}); retrying as update")
+        body = self._make_post(url, {"update-requesthandler": handler_def})
+        if body.get("errorMessages"):
+            raise SolrAPIError(f"Failed to upsert request handler {name}: {body['errorMessages']}")
+        if self.logging_verbose >= 1:
+            logger.info(f"update-requesthandler {name} on collection {self.collection}")
+
     def delete_collection(self):
         """Delete a Solr collection."""
         url = urljoin(self.base_url, f"api/collections/{self.collection}")

search/test_solrapi.py

@@ -0,0 +1,66 @@
+from unittest import mock
+
+import pytest
+
+from search.solrapi import SolrAPIError, SolrManagementAPI
+
+
+def _mock_response(json_payload):
+    resp = mock.Mock()
+    resp.json.return_value = json_payload
+    resp.raise_for_status = mock.Mock()
+    return resp
+
+
+@mock.patch("search.solrapi.requests")
+def test_upsert_request_handler_adds_when_absent(mock_requests):
+    # Successful add → 200 with no errorMessages.
+    mock_requests.post.return_value = _mock_response({})
+
+    api = SolrManagementAPI("http://search:8983", "freesound1234")
+    api.upsert_request_handler("/select_similarity", "solr.SearchHandler")
+
+    assert mock_requests.post.call_count == 1
+    _, kwargs = mock_requests.post.call_args
+    assert kwargs["json"] == {"add-requesthandler": {"name": "/select_similarity", "class": "solr.SearchHandler"}}
+
+
+@mock.patch("search.solrapi.requests")
+def test_upsert_request_handler_falls_back_to_update_on_duplicate(mock_requests):
+    # Solr returns 200 with errorMessages when the handler already exists; we
+    # then retry with update, which succeeds.
+    mock_requests.post.side_effect = [
+        _mock_response({"errorMessages": [{"add-requesthandler": "'/select_similarity' already exists"}]}),
+        _mock_response({}),
+    ]
+
+    api = SolrManagementAPI("http://search:8983", "freesound1234")
+    api.upsert_request_handler("/select_similarity", "solr.SearchHandler")
+
+    assert mock_requests.post.call_count == 2
+    second_call_json = mock_requests.post.call_args_list[1].kwargs["json"]
+    assert second_call_json == {"update-requesthandler": {"name": "/select_similarity", "class": "solr.SearchHandler"}}
+
+
+@mock.patch("search.solrapi.requests")
+def test_upsert_request_handler_raises_when_update_also_fails(mock_requests):
+    # Both add and update return errorMessages → surface as SolrAPIError.
+    mock_requests.post.side_effect = [
+        _mock_response({"errorMessages": [{"add-requesthandler": "boom"}]}),
+        _mock_response({"errorMessages": [{"update-requesthandler": "boom too"}]}),
+    ]
+
+    api = SolrManagementAPI("http://search:8983", "freesound1234")
+    with pytest.raises(SolrAPIError):
+        api.upsert_request_handler("/select_similarity", "solr.SearchHandler")
+
+
+@mock.patch("search.solrapi.requests")
+def test_upsert_request_handler_includes_defaults(mock_requests):
+    mock_requests.post.return_value = _mock_response({})
+
+    api = SolrManagementAPI("http://search:8983", "freesound1234")
+    api.upsert_request_handler("/select_similarity", "solr.SearchHandler", defaults={"defType": "dismax"})
+
+    _, kwargs = mock_requests.post.call_args
+    assert kwargs["json"]["add-requesthandler"]["defaults"] == {"defType": "dismax"}

utils/search/backends/solr555pysolr.py

@@ -687,6 +687,8 @@ def search_sounds(
                 if similar_to is not None
                 else settings.SEARCH_SOLR_TIMEOUT_SECONDS
             )
+        # Use a separate request handler for similarity to have separate metrics
+        search_handler = "select_similarity" if similar_to is not None else None
 
         if field_list is None:
             # We generally only want the sound IDs of the results as we load data from DB
@@ -890,7 +892,9 @@ def search_sounds(
                 if not group_counts_as_one_in_facets and "json.facet" in query_as_kwargs:
                     facets_kwarg = query_as_kwargs["json.facet"]
                     query_as_kwargs["json.facet"] = {}
-                results = self.get_sounds_index(timeout=timeout).search(**query_as_kwargs)
+                results = self.get_sounds_index(timeout=timeout, search_handler=search_handler).search(
+                    **query_as_kwargs
+                )
 
                 # Now make the second query in which we get the facets and the total number of results, and remove the "collapse" filter.
                 fq = [fq_element for fq_element in query_as_kwargs["fq"] if "collapse field" not in fq_element]
@@ -899,13 +903,17 @@ def search_sounds(
                 query_as_kwargs["expand"] = False
                 if not group_counts_as_one_in_facets and facets_kwarg is not None:
                     query_as_kwargs["json.facet"] = facets_kwarg
-                results_extra_query = self.get_sounds_index(timeout=timeout).search(**query_as_kwargs)
+                results_extra_query = self.get_sounds_index(timeout=timeout, search_handler=search_handler).search(
+                    **query_as_kwargs
+                )
                 if not group_counts_as_one_in_facets:
                     results.facets = results_extra_query.facets
                 results.non_grouped_number_of_results = results_extra_query.num_found
             else:
                 # If we are not using collapse and expand query parser (and/or not grouping by pack), just run the query.
-                results = self.get_sounds_index(timeout=timeout).search(**query_as_kwargs)
+                results = self.get_sounds_index(timeout=timeout, search_handler=search_handler).search(
+                    **query_as_kwargs
+                )
 
             # Facets returned in results use the corresponding solr fieldnames as keys. We want to convert them to the
             # original fieldnames so that the rest of the code can use them without knowing about the solr fieldnames.

utils/search/backends/solr9pysolr.py

@@ -41,13 +41,16 @@ def __init__(self, sounds_index_url=None, forum_index_url=None):
         self.forum_index_url = forum_index_url
         self.solr_base_url = settings.SOLR9_BASE_URL
 
-    def get_sounds_index(self, timeout=settings.SEARCH_SOLR_TIMEOUT_SECONDS):
+    def get_sounds_index(self, timeout=settings.SEARCH_SOLR_TIMEOUT_SECONDS, search_handler=None):
+        # Only forward search_handler when set
+        extra = {"search_handler": search_handler} if search_handler is not None else {}
         return pysolr.Solr(
             self.sounds_index_url,
             encoder=solr555pysolr.FreesoundSoundJsonEncoder(),
             results_cls=solr555pysolr.SolrResponseInterpreter,
             always_commit=True,
             timeout=timeout,
+            **extra,
         )
 
     def get_forum_index(self, timeout=settings.SEARCH_SOLR_TIMEOUT_SECONDS):

utils/search/schema/freesound.json

@@ -531,5 +531,11 @@
         "type_facet"
       ]
     }
+  ],
+  "requestHandlers": [
+    {
+      "name": "/select_similarity",
+      "class": "solr.SearchHandler"
+    }
   ]
 }

utils/tests/test_solr_query.py

@@ -1,7 +1,10 @@
+from unittest import mock
+
 import pytest
 from django.conf import settings
 
 from utils.search import SearchEngineException, SearchEngineTimeoutException
+from utils.search.backends.solr9pysolr import Solr9PySolrSearchEngine
 from utils.search.backends.solr555pysolr import Solr555PySolrSearchEngine
 from utils.search.backends.solr_common import SolrQuery, SolrResponseInterpreter
 
@@ -108,3 +111,19 @@ def test_solr_response_partial_results_grouped_missing_ngroups():
     }
     with pytest.raises(SearchEngineTimeoutException):
         SolrResponseInterpreter(response)
+
+
+@mock.patch("utils.search.backends.solr9pysolr.pysolr.Solr")
+def test_get_sounds_index_default_omits_search_handler(mock_solr):
+    # When no search_handler is provided, we don't pass it to pysolr at all so
+    # pysolr's own default kicks in (the falsy-fallback to "select").
+    Solr9PySolrSearchEngine().get_sounds_index()
+    _, kwargs = mock_solr.call_args
+    assert "search_handler" not in kwargs
+
+
+@mock.patch("utils.search.backends.solr9pysolr.pysolr.Solr")
+def test_get_sounds_index_passes_explicit_search_handler(mock_solr):
+    Solr9PySolrSearchEngine().get_sounds_index(search_handler="select_similarity")
+    _, kwargs = mock_solr.call_args
+    assert kwargs["search_handler"] == "select_similarity"