Remove public_scan_b field

openlibrary/core/models.py

@@ -705,7 +705,7 @@ def get_ebook_info(self):
         """
         solrdata = web.storage(self._solr_data or {})
         d = {}
-        if solrdata.get("has_fulltext") and solrdata.get("public_scan_b"):
+        if solrdata.get("has_fulltext") and solrdata.get("ebook_access") == "public":
             d["read_url"] = f"https://archive.org/stream/{solrdata.ia[0]}"
             d["has_ebook"] = True
         elif solrdata.get("lending_edition_s"):

openlibrary/fastapi/search.py

@@ -55,16 +55,12 @@ class PublicQueryOptions(BaseModel):
     place: str | None = None
     person: str | None = None
     time: str | None = None
-    # from workscheme facet_fields
+
+    # Legacy availability fields ; handled by WorkSearchScheme. Need to be specified
+    # here or else will be ignored.
     has_fulltext: Literal["true", "false"] | None = None
-    public_scan_b: list[Literal["true", "false"]] = []
-    # Availability filters. These are not facet_fields — they're handled by
-    # WorkSearchScheme.facet_rewrites, which maps (`public_scan`, "true") and
-    # (`print_disabled`, "true") to `ebook_access:*` fq clauses. Mirrors the
-    # explicit whitelist in the web.py /search handler (worksearch/code.py);
-    # without declaring them here, FastAPI silently drops the params and the
-    # header search modal's availability filters do nothing.
     public_scan: Literal["true", "false"] | None = None
+    public_scan_b: list[Literal["true", "false"]] = []
     print_disabled: Literal["true", "false"] | None = None
 
     # List fields (facets)

openlibrary/i18n/messages.pot

@@ -6710,14 +6710,6 @@ msgstr ""
 msgid "Published by"
 msgstr ""
 
-#: search/work_search_selected_facets.html
-msgid "Only Classic eBooks"
-msgstr ""
-
-#: search/work_search_selected_facets.html
-msgid "Classic eBooks hidden"
-msgstr ""
-
 #: search/work_search_selected_facets.html
 #, python-format
 msgid "%(title)s - search"
@@ -8752,7 +8744,3 @@ msgstr ""
 msgid "First published"
 msgstr ""
 
-#: openlibrary/plugins/worksearch/code.py
-msgid "Classic eBooks"
-msgstr ""
-

openlibrary/plugins/openlibrary/api.py

@@ -744,7 +744,7 @@ def build_homepage():
                     Catalog.create(
                         metadata=Metadata(title=_("Classic Books")),
                         response=provider.search(
-                            query='ddc:8* first_publish_year:[* TO 1950] publish_year:[2000 TO *] NOT public_scan_b:false -subject:"content_warning:cover"',
+                            query='ddc:8* first_publish_year:[* TO 1950] publish_year:[2000 TO *] ebook_access:public -subject:"content_warning:cover"',
                             sort="trending",
                             limit=25,
                         ),

openlibrary/plugins/upstream/models.py

@@ -502,7 +502,7 @@ def _solr_data(self):
             "first_publish_year",
             "has_fulltext",
             "lending_edition_s",
-            "public_scan_b",
+            "ebook_access",
         ] + get_solr_keys()
         solr = get_solr()
         stats.begin("solr", get=self.key, fields=fields)

openlibrary/plugins/worksearch/code.py

@@ -116,7 +116,6 @@ def get_facet_map() -> tuple[tuple[str, str]]:
         ("person_facet", _("People")),
         ("place_facet", _("Places")),
         ("time_facet", _("Times")),
-        ("public_scan_b", _("Classic eBooks")),
     )
 
 
@@ -658,7 +657,7 @@ def get_doc(doc: SolrDocument):
         ia=doc.get("ia", []),
         collections=(doc.get("ia_collection") or []),
         has_fulltext=doc.get("has_fulltext", False),
-        public_scan=doc.get("public_scan_b", bool(doc.get("ia"))),
+        public_scan=doc.get("ebook_access") == "public",
         lending_edition=doc.get("lending_edition_s", None),
         lending_identifier=doc.get("lending_identifier_s", None),
         authors=[
@@ -783,7 +782,6 @@ def GET(self):
             person_facet=[],
             place_facet=[],
             time_facet=[],
-            public_scan_b=[],
         )
 
         # Send to full-text Search Inside if checkbox checked

openlibrary/plugins/worksearch/schemes/works.py

@@ -77,6 +77,7 @@ class WorkSearchScheme(SearchScheme):
             "place",
             "time",
             "has_fulltext",
+            "public_scan_b",
             "title_suggest",
             "publish_year",
             "language",
@@ -129,7 +130,6 @@ class WorkSearchScheme(SearchScheme):
             "person_facet",
             "place_facet",
             "time_facet",
-            "public_scan_b",
         }
     )
     field_name_map = MappingProxyType(
@@ -211,7 +211,6 @@ class WorkSearchScheme(SearchScheme):
             "first_publish_year",
             "cover_i",
             "cover_edition_key",
-            "public_scan_b",
             "lending_edition_s",
             "lending_identifier_s",
             "language",
@@ -231,6 +230,8 @@ class WorkSearchScheme(SearchScheme):
         {
             ("public_scan", "true"): "ebook_access:public",
             ("public_scan", "false"): "-ebook_access:public",
+            ("public_scan_b", "true"): "ebook_access:public",
+            ("public_scan_b", "false"): "-ebook_access:public",
             ("print_disabled", "true"): "ebook_access:printdisabled",
             ("print_disabled", "false"): "-ebook_access:printdisabled",
             (
@@ -280,13 +281,28 @@ def transform_user_query(self, user_query: str, q_tree: luqum.tree.Item) -> luqu
                 has_search_fields = True
                 if node.name.lower() in self.field_name_map:
                     node.name = self.field_name_map[node.name.lower()]
+
                 if node.name == "isbn":
                     isbn_transform(node)
-                if node.name in ("lcc", "lcc_sort"):
+                elif node.name in ("lcc", "lcc_sort"):
                     lcc_transform(node)
-                if node.name in ("dcc", "dcc_sort"):
+                elif node.name in ("dcc", "dcc_sort"):
                     ddc_transform(node)
 
+                field_val = node.children[0]
+                if isinstance(field_val, (luqum.tree.Word, luqum.tree.Phrase)):
+                    val = str(field_val.value).strip('"').lower()
+                    if rewrite := self.facet_rewrites.get((node.name, val)):
+                        if callable(rewrite):
+                            rewrite = rewrite()
+                        if not rewrite.startswith("("):
+                            rewrite = f"({rewrite})"
+                        new_node = luqum_parser(rewrite)
+                        if parents:
+                            luqum_replace_child(parents[-1], node, new_node)
+                        else:
+                            q_tree = new_node
+
         if not has_search_fields:
             # If there are no search fields, maybe we want just an isbn?
             isbn = normalize_isbn(user_query)
@@ -413,7 +429,6 @@ def remove_work_prefix(field: str) -> str:
                 "ia": "ia",
                 "ia_collection": "ia_collection",
                 "ia_box_id": "ia_box_id",
-                "public_scan_b": "public_scan_b",
             }
 
             def convert_work_field_to_edition_field(

openlibrary/plugins/worksearch/subjects.py

@@ -30,12 +30,9 @@ def GET(self, key):
         if (nkey := self.normalize_key(key)) != key:
             raise web.redirect(nkey)
 
-        # this needs to be updated to include:
-        # q=public_scan_b:true+OR+lending_edition_s:*
         subj = get_subject(
             key,
             details=True,
-            filters={"public_scan_b": "false", "lending_edition_s": "*"},
             sort=web.input(sort="readinglog").sort,
             request_label="SUBJECT_ENGINE_PAGE",
         )
@@ -247,7 +244,7 @@ async def get_subject_async(
                 "has_fulltext",
                 "subject",
                 "ia_collection",
-                "public_scan_b",
+                "ebook_access",
                 "lending_edition_s",
                 "lending_identifier_s",
             ],
@@ -379,7 +376,7 @@ def work_wrapper(w: dict) -> web.storage:
             authors=[web.storage(key=f"/authors/{olid}", name=name) for olid, name in zip(w.get("author_key", []), w.get("author_name", []))],
             first_publish_year=w.get("first_publish_year"),
             ia=w.get("ia", [None])[0],
-            public_scan=w.get("public_scan_b", bool(w.get("ia"))),
+            public_scan=w.get("ebook_access") == "public",
             has_fulltext=w.get("has_fulltext", False),
         )
 

openlibrary/plugins/worksearch/tests/test_worksearch.py

@@ -32,7 +32,6 @@ def test_get_doc():
             "ia": ["computerglossary00free"],
             "key": "/works/OL1820355W",
             "lending_edition_s": "OL1111795M",
-            "public_scan_b": False,
             "title": "The computer glossary",
             "ratings_average": None,
             "ratings_count": None,

openlibrary/solr/query_utils.py

@@ -34,10 +34,35 @@ def luqum_remove_child(child: Item, parents: list[Item]):
         raise NotImplementedError(f"Not implemented for Item subclass: {parent.__class__.__name__}")
 
 
+def luqum_get_head(item: Item) -> str:
+    """
+    Whitespace could be attached anywhere recursively
+    """
+    head = item.head
+    if item.children:
+        head += luqum_get_head(item.children[0])
+    return head
+
+
+def luqum_get_tail(item: Item) -> str:
+    """
+    Whitespace could be attached anywhere recursively
+    """
+    tail = item.tail
+    if item.children:
+        tail = luqum_get_tail(item.children[-1]) + tail
+    return tail
+
+
 def luqum_replace_child(parent: Item, old_child: Item, new_child: Item):
     """
     Replaces a child in a luqum parse tree.
     """
+    # Preserve the old child's surrounding whitespace so re-serializing the
+    # tree doesn't concatenate tokens together.
+    new_child.head = luqum_get_head(old_child)
+    new_child.tail = luqum_get_tail(old_child)
+
     if isinstance(parent, (BaseOperation, Group, Unary)):
         new_children = tuple(new_child if c == old_child else c for c in parent.children)
         parent.children = new_children

openlibrary/solr/solr_types.py

@@ -134,7 +134,6 @@ class SolrDocument(TypedDict):
     last_modified: Optional[str]
     seed_count: Optional[int]
     list_type: Optional[str]
-    public_scan_b: Optional[bool]
     printdisabled_s: Optional[str]
     lending_edition_s: Optional[str]
     ebook_count_i: Optional[int]

openlibrary/solr/types_generator.py

@@ -5,7 +5,6 @@
 root = os.path.dirname(__file__)
 OVERRIDES = {
     "type": "Literal['work', 'author', 'subject']",
-    "public_scan_b": "Optional[bool]",
     "printdisabled_s": "Optional[str]",
     "lending_edition_s": "Optional[str]",
     "ebook_count_i": "Optional[int]",

openlibrary/solr/updater/edition.py

@@ -347,10 +347,6 @@ def ebook_provider(self) -> list[str]:
     def has_fulltext(self) -> bool:
         return self.ebook_access > bp.EbookAccess.UNCLASSIFIED
 
-    @property
-    def public_scan_b(self) -> bool:
-        return self.ebook_access == bp.EbookAccess.PUBLIC
-
     def build(self) -> SolrDocument:
         """
         Build the solr document for the given edition to store as a nested
@@ -404,7 +400,6 @@ def build(self) -> SolrDocument:
                 "ebook_access": self.ebook_access.to_solr_str(),
                 "ebook_provider": self.ebook_provider,
                 "has_fulltext": self.has_fulltext,
-                "public_scan_b": self.public_scan_b,
             },
         )
 

openlibrary/solr/updater/work.py

@@ -518,10 +518,6 @@ def ebook_provider(self) -> list[str]:
     def has_fulltext(self) -> bool:
         return any(e.has_fulltext for e in self._solr_editions)
 
-    @property
-    def public_scan_b(self) -> bool:
-        return any(e.public_scan_b for e in self._solr_editions)
-
     @cached_property
     def ia(self) -> list[str]:
         return [cast(str, e.ia) for e in self._ia_editions]

openlibrary/templates/home/index.html

@@ -20,7 +20,7 @@
 
   $if not test:
     $:macros.QueryCarousel(query='trending_score_hourly_sum:[1 TO *] readinglog_count:[4 TO *]', title=_('Trending Books'), key="trending", sort='trending', has_fulltext_only=False, user_lang_only=True)
-    $:macros.QueryCarousel(query="ddc:8* first_publish_year:[* TO 1950] publish_year:[2000 TO *] NOT public_scan_b:false", title=_('Classic Books'), key="public_domain", sort='trending', user_lang_only=True)
+    $:macros.QueryCarousel(query="ddc:8* first_publish_year:[* TO 1950] publish_year:[2000 TO *] ebook_access:public", title=_('Classic Books'), key="public_domain", sort='trending', user_lang_only=True)
 
   $:render_template("home/custom_ia_carousel", title=_('Books We Love'), key="staff_picks", subject="openlibrary_staff_picks", sorts=["lending___last_browse desc"], limit=18, test=test, user_lang_only=True)
 

openlibrary/templates/search/work_search_facets.html

@@ -81,8 +81,7 @@ <h3 class="collapse">$_("Zoom In")</h3>
         $# the filter row now, so it's no longer offered as a sidebar facet.
         $if header=='has_fulltext':
             $continue
-        $if header=='public_scan_b':
-            $continue
+
         $ counts = [i for i in facet_counts[header] if i[0] not in param.get(header, [])] if not async_load else [(None, None, None)]
         $if len(counts) <= 1 and not async_load:
             $continue

openlibrary/templates/search/work_search_selected_facets.html

@@ -18,7 +18,7 @@
     # header search modal: `has_fulltext` / `public_scan` / `print_disabled` by
     # the availability toggle, and `language` by the language popover. Excluded
     # from the facet_map chip loop below.
-    special_handled = {'has_fulltext', 'public_scan_b', 'language'}
+    special_handled = {'has_fulltext', 'language'}
 
     def del_facet_url(k, v):
         if k != 'has_fulltext':
@@ -99,15 +99,6 @@
                     $ chip_display = fulltext_names.get(k, '')
                     <ol-chip selected size="small" href="$del_facet_url('has_fulltext', k)">$chip_display</ol-chip>
 
-            $if 'public_scan_b' in param:
-                $ counts = search_response.facet_counts.get('public_scan_b', [])
-                $for k, display, count in counts:
-                    $if k not in param.get('public_scan_b', []):
-                        $continue
-                    $# TODO: Consider removing public_scan_b handling. No UI exposes
-                    $# this facet (the sidebar skips it), so it only triggers via
-                    $# manually crafted URLs like /search?public_scan_b=true.
-                    $ chip_display = _("Only Classic eBooks") if display == 'true' else _("Classic eBooks hidden")
-                    <ol-chip selected size="small" href="$del_facet_url('public_scan_b', k)">$chip_display</ol-chip>
+
         </div>
     $var title: $_('%(title)s - search', title=', '.join(title))

openlibrary/templates/work_search.html

@@ -30,7 +30,7 @@ <h1>$_("Search Books")</h1>
                 $:_('Solr Editions')
             </label>
         </div>
-      $ sticky = set(['sort', 'author_facet', 'language', 'first_publish_year', 'publisher_facet',  'subject_facet', 'person_facet', 'place_facet', 'time_facet', 'public_scan_b', 'has_fulltext', 'public_scan', 'print_disabled'])
+      $ sticky = set(['sort', 'author_facet', 'language', 'first_publish_year', 'publisher_facet',  'subject_facet', 'person_facet', 'place_facet', 'time_facet', 'has_fulltext', 'public_scan', 'print_disabled'])
 
       $for k, values in param.items():
         $if k not in sticky:

openlibrary/tests/solr/test_query_utils.py

@@ -39,13 +39,13 @@ def fn(query: str, remove: str) -> str:
         "title:foo OR id:1",
         "title:foo",
         "(title:foo OR bar:foo)",
-        "(title:foo OR bar:foo)OR id:1",
+        "(title:foo OR bar:foo) OR id:1",
     ),
     "Deeply nested": (
         "title:foo OR (id:1 OR id:2)",
         "id:2",
         "(subject:horror)",
-        "title:foo OR (id:1 OR(subject:horror))",
+        "title:foo OR (id:1 OR (subject:horror))",
     ),
 }
 

openlibrary/tests/solr/updater/test_work.py

@@ -190,7 +190,6 @@ def test_with_one_lending_edition(self):
         )
 
         assert d.has_fulltext is True
-        assert d.public_scan_b is False
         assert d.printdisabled_s is None
         assert d.lending_edition_s == "OL1M"
         assert d.ia == ["foo00bar"]
@@ -212,7 +211,6 @@ def test_with_two_lending_editions(self):
             },
         )
         assert d.has_fulltext is True
-        assert d.public_scan_b is False
         assert d.printdisabled_s is None
         assert d.lending_edition_s == "OL1M"
         assert sorted(d.ia) == ["foo01bar", "foo02bar"]
@@ -228,7 +226,6 @@ def test_with_one_inlibrary_edition(self):
             ia_metadata={"foo00bar": {"collection": ["printdisabled", "inlibrary"]}},
         )
         assert d.has_fulltext is True
-        assert d.public_scan_b is False
         assert d.printdisabled_s == "OL1M"
         assert d.lending_edition_s == "OL1M"
         assert d.ia == ["foo00bar"]
@@ -244,7 +241,6 @@ def test_with_one_printdisabled_edition(self):
             ia_metadata={"foo00bar": {"collection": ["printdisabled", "americana"]}},
         )
         assert d.has_fulltext is True
-        assert d.public_scan_b is False
         assert d.printdisabled_s == "OL1M"
         assert d.lending_edition_s is None
         assert d.ia == ["foo00bar"]
@@ -283,7 +279,6 @@ def test_with_multiple_editions(self):
             },
         )
         assert d.has_fulltext is True
-        assert d.public_scan_b is True
         assert d.printdisabled_s == "OL4M"
         assert d.lending_edition_s == "OL2M"
         assert sorted(d.ia) == ["foo00bar", "foo01bar", "foo02bar"]