Merge pull request #11312 from cdrini/refactor/subject-engine

Label more solr queries for monitoring + some SubjectEngine, QueryLabel refactoring

Author: jimchamp

openlibrary/macros/RawQueryCarousel.html

@@ -60,7 +60,7 @@
       if has_fulltext_only:
         params['has_fulltext'] = 'true'
 
-      results = work_search(params, fields = ",".join(fields), sort=sort, limit=limit, facet=False, query_label='BOOK_CAROUSEL')
+      results = work_search(params, fields = ",".join(fields), sort=sort, limit=limit, facet=False, request_label='BOOK_CAROUSEL')
       books = [storage(b) for b in (results.get('docs', []))]
       load_more = {
         "queryType": "SEARCH",

openlibrary/plugins/openlibrary/partials.py

@@ -239,7 +239,7 @@ def generate(self) -> dict:
             rows=0,
             spellcheck_count=3,
             facet=True,
-            query_label='BOOK_SEARCH_FACETS',
+            request_label='BOOK_SEARCH_FACETS',
         )
 
         sidebar = render_template(

openlibrary/plugins/upstream/models.py

@@ -528,6 +528,7 @@ def get_books(self, q=''):
             has_fulltext=i.mode == "ebooks",
             query=q,
             facet=True,
+            request_label='AUTHOR_BOOKS_PAGE',
         )
 
     def get_work_count(self):
@@ -603,7 +604,7 @@ def _solr_data(self):
         solr = get_solr()
         stats.begin("solr", get=self.key, fields=fields)
         try:
-            return solr.get(self.key, fields=fields)
+            return solr.get(self.key, fields=fields, request_label='GET_WORK_SOLR_DATA')
         except Exception:
             logging.getLogger("openlibrary").exception("Failed to get solr data")
             return None

openlibrary/plugins/worksearch/code.py

@@ -45,6 +45,7 @@
 from openlibrary.utils.solr import (
     DEFAULT_PASS_TIME_ALLOWED,
     DEFAULT_SOLR_TIMEOUT_SECONDS,
+    SolrRequestLabel,
 )
 
 logger = logging.getLogger("openlibrary.worksearch")
@@ -195,24 +196,6 @@ def read_cookie():
     return 'details'
 
 
-QueryLabel = Literal[
-    'UNLABELLED',
-    'BOOK_SEARCH',
-    'BOOK_SEARCH_API',
-    'BOOK_SEARCH_FACETS',
-    'BOOK_CAROUSEL',
-    # Used for the internal request made by solr to choose the best edition
-    # during a normal book search
-    'EDITION_MATCH',
-    'LIST_SEARCH',
-    'LIST_SEARCH_API',
-    'SUBJECT_SEARCH',
-    'SUBJECT_SEARCH_API',
-    'AUTHOR_SEARCH',
-    'AUTHOR_SEARCH_API',
-]
-
-
 def run_solr_query(  # noqa: PLR0912
     scheme: SearchScheme,
     param: dict | None = None,
@@ -225,7 +208,7 @@ def run_solr_query(  # noqa: PLR0912
     facet: bool | Iterable[str] = True,
     allowed_filter_params: set[str] | None = None,
     extra_params: list[tuple[str, Any]] | None = None,
-    query_label: QueryLabel = 'UNLABELLED',
+    request_label: SolrRequestLabel = 'UNLABELLED',
 ):
     """
     :param param: dict of query parameters
@@ -245,7 +228,7 @@ def run_solr_query(  # noqa: PLR0912
         *(('fq', subquery) for subquery in scheme.universe),
         ('start', offset),
         ('rows', rows),
-        ('ol.label', query_label),
+        ('ol.label', request_label),
         ('wt', param.get('wt', 'json')),
     ] + (extra_params or [])
 
@@ -388,7 +371,7 @@ def do_search(
     rows=100,
     facet=False,
     spellcheck_count=None,
-    query_label: QueryLabel = 'UNLABELLED',
+    request_label: SolrRequestLabel = 'UNLABELLED',
 ):
     """
     :param param: dict of search url parameters
@@ -419,7 +402,7 @@ def do_search(
         spellcheck_count,
         fields=list(fields),
         facet=facet,
-        query_label=query_label,
+        request_label=request_label,
     )
 
 
@@ -619,7 +602,7 @@ def GET(self):
                 page,
                 rows=rows,
                 spellcheck_count=3,
-                query_label='BOOK_SEARCH',
+                request_label='BOOK_SEARCH',
             )
         else:
             search_response = SearchResponse(
@@ -643,6 +626,7 @@ def works_by_author(
     facet=False,
     has_fulltext=False,
     query: str | None = None,
+    request_label: SolrRequestLabel = 'UNLABELLED',
 ):
     param = {'q': query or '*:*'}
     if has_fulltext:
@@ -663,6 +647,7 @@ def works_by_author(
                 "time_facet",
             ]
         ),
+        request_label=request_label,
         fields=list(
             WorkSearchScheme.default_fetched_fields | {'editions', 'providers'}
         ),
@@ -750,7 +735,7 @@ def GET(self):  # referenced subject_search
     def get_results(
         self,
         req: ListSearchRequest,
-        query_label: Literal['LIST_SEARCH', 'LIST_SEARCH_API'],
+        request_label: Literal['LIST_SEARCH', 'LIST_SEARCH_API'],
     ):
         return run_solr_query(
             ListSearchScheme(),
@@ -759,7 +744,7 @@ def get_results(
             rows=req.limit,
             fields=req.fields,
             sort=req.sort,
-            query_label=query_label,
+            request_label=request_label,
         )
 
 
@@ -808,13 +793,15 @@ class subject_search(delegate.page):
     path = '/search/subjects'
 
     def GET(self):
-        get_results = functools.partial(self.get_results, query_label='SUBJECT_SEARCH')
+        get_results = functools.partial(
+            self.get_results, request_label='SUBJECT_SEARCH'
+        )
         return render_template('search/subjects', get_results)
 
     def get_results(
         self,
         q,
-        query_label: Literal['SUBJECT_SEARCH', 'SUBJECT_SEARCH_API'],
+        request_label: Literal['SUBJECT_SEARCH', 'SUBJECT_SEARCH_API'],
         offset=0,
         limit=100,
     ):
@@ -824,7 +811,7 @@ def get_results(
             offset=offset,
             rows=limit,
             sort='work_count desc',
-            query_label=query_label,
+            request_label=request_label,
         )
 
         return response
@@ -842,7 +829,7 @@ def GET(self):
 
         response = self.get_results(
             i.q,
-            query_label='SUBJECT_SEARCH_API',
+            request_label='SUBJECT_SEARCH_API',
             offset=offset,
             limit=limit,
         )
@@ -861,13 +848,13 @@ class author_search(delegate.page):
     path = '/search/authors'
 
     def GET(self):
-        get_results = functools.partial(self.get_results, query_label='AUTHOR_SEARCH')
+        get_results = functools.partial(self.get_results, request_label='AUTHOR_SEARCH')
         return render_template('search/authors', get_results)
 
     def get_results(
         self,
         q,
-        query_label: Literal['AUTHOR_SEARCH', 'AUTHOR_SEARCH_API'],
+        request_label: Literal['AUTHOR_SEARCH', 'AUTHOR_SEARCH_API'],
         offset=0,
         limit=100,
         fields='*',
@@ -880,7 +867,7 @@ def get_results(
             rows=limit,
             fields=fields,
             sort=sort,
-            query_label=query_label,
+            request_label=request_label,
         )
 
         return resp
@@ -898,7 +885,7 @@ def GET(self):
 
         response = self.get_results(
             i.q,
-            query_label='AUTHOR_SEARCH_API',
+            request_label='AUTHOR_SEARCH_API',
             offset=offset,
             limit=limit,
             fields=i.fields,
@@ -968,7 +955,7 @@ def work_search(
     fields: str = '*',
     facet: bool = True,
     spellcheck_count: int | None = None,
-    query_label: QueryLabel = 'UNLABELLED',
+    request_label: SolrRequestLabel = 'UNLABELLED',
 ) -> dict:
     """
     :param sort: key of SORTS dict at the top of this file
@@ -991,7 +978,7 @@ def work_search(
         fields=fields,
         facet=facet,
         spellcheck_count=spellcheck_count,
-        query_label=query_label,
+        request_label=request_label,
     )
     response = resp.raw_resp['response']
 
@@ -1065,7 +1052,7 @@ def GET(self):
             # so disable it. This makes it much faster.
             facet=False,
             spellcheck_count=spellcheck_count,
-            query_label='BOOK_SEARCH_API',
+            request_label='BOOK_SEARCH_API',
         )
         response['documentation_url'] = "https://openlibrary.org/dev/docs/api/search"
         response['q'] = q

openlibrary/plugins/worksearch/languages.py

@@ -2,7 +2,8 @@
 
 import json
 import logging
-from typing import Literal
+from dataclasses import dataclass
+from typing import Literal, override
 
 import web
 
@@ -140,7 +141,15 @@ def process(p):
         return [process(p) for p in language_facets]
 
 
+@dataclass
 class LanguageEngine(subjects.SubjectEngine):
+    name: str = "language"
+    key: str = "languages"
+    prefix: str = "/languages/"
+    facet: str = "language"
+    facet_key: str = "language"
+
+    @override
     def normalize_key(self, key):
         return key
 
@@ -160,13 +169,4 @@ def get_ebook_count(self, name, value, publish_year):
 
 
 def setup():
-    subjects.SUBJECTS.append(
-        subjects.SubjectMeta(
-            name="language",
-            key="languages",
-            prefix="/languages/",
-            facet="language",
-            facet_key="language",
-            Engine=LanguageEngine,
-        )
-    )
+    subjects.SUBJECTS.append(LanguageEngine())

openlibrary/plugins/worksearch/publishers.py

@@ -1,6 +1,8 @@
 """Publisher pages"""
 
 import logging
+from dataclasses import dataclass
+from typing import override
 
 import web
 
@@ -17,7 +19,11 @@ class publishers(subjects.subjects):
 
     def GET(self, key):
         key = key.replace("_", " ")
-        page = subjects.get_subject(key, details=True)
+        page = subjects.get_subject(
+            key,
+            details=True,
+            request_label='SUBJECT_ENGINE_PAGE',
+        )
 
         if not page or page.work_count == 0:
             web.ctx.status = "404 Not Found"
@@ -82,7 +88,15 @@ def process_result(self, result):
         ]
 
 
+@dataclass
 class PublisherEngine(subjects.SubjectEngine):
+    name: str = "publisher"
+    key: str = "publishers"
+    prefix: str = "/publishers/"
+    facet: str = "publisher_facet"
+    facet_key: str = "publisher_facet"
+
+    @override
     def normalize_key(self, key):
         return key
 
@@ -102,13 +116,4 @@ def get_ebook_count(self, name, value, publish_year):
 
 
 def setup():
-    subjects.SUBJECTS.append(
-        subjects.SubjectMeta(
-            name="publisher",
-            key="publishers",
-            prefix="/publishers/",
-            facet="publisher_facet",
-            facet_key="publisher_facet",
-            Engine=PublisherEngine,
-        )
-    )
+    subjects.SUBJECTS.append(PublisherEngine())