Merge pull request #678 from scieloorg/ajust-match-performance

Ajusta desempenho de match por DOI e insere filtro collection em fixture

Author: pitangainnovare

etl/deduplicator/openalex.py

@@ -131,7 +131,18 @@ def _search_openalex_by_doi(
             logger.warning("Invalid DOI after normalization: %s", doi)
             return []
 
-        query = {"bool": {"filter": [{"wildcard": {"doi.keyword": f"*{normalized_doi}*"}}]}}
+        query = {
+            "bool": {
+                "filter": [
+                    {
+                        "bool": {
+                            "should": self._doi_exact_or_prefix_queries(normalized_doi),
+                            "minimum_should_match": 1,
+                        }
+                    }
+                ]
+            }
+        }
         if year is not None:
             try:
                 year = int(year)
@@ -151,6 +162,23 @@ def _search_openalex_by_doi(
             logger.error("Error searching OpenAlex by DOI: %s", exc)
             return []
 
+    def _doi_exact_or_prefix_queries(self, normalized_doi: str) -> list[dict[str, Any]]:
+        doi_values = [
+            normalized_doi,
+            f"https://doi.org/{normalized_doi}",
+            f"http://doi.org/{normalized_doi}",
+            f"https://dx.doi.org/{normalized_doi}",
+            f"http://dx.doi.org/{normalized_doi}",
+        ]
+        fields = ["doi.keyword", "ids.doi.keyword"]
+
+        queries: list[dict[str, Any]] = []
+        for field in fields:
+            queries.extend({"term": {field: value}} for value in doi_values)
+            queries.extend({"prefix": {field: value}} for value in doi_values)
+
+        return queries
+
     def _search_openalex_by_isbn(
         self,
         isbns: List[str],

etl/tests/pipeline/test_pipeline_dedup.py

@@ -214,6 +214,30 @@ def test_openalex_doi_match_keeps_language_variants_with_same_normalized_doi(sel
         )
         self.assertTrue(all(match[1] == "doi" for match in matches))
 
+    def test_openalex_doi_search_uses_exact_or_prefix_queries(self):
+        matcher = make_matcher("article")
+        matcher.input_openalex_index = "raw_openalex_works"
+        matcher.client = Mock()
+        matcher.client.client.search.return_value = {"hits": {"hits": []}}
+
+        matcher._search_openalex_by_doi(
+            "10.1590/0034-7167.202578SUPL101",
+            {"publication_year": 2025},
+        )
+
+        body = matcher.client.client.search.call_args.kwargs["body"]
+        self.assertNotIn("wildcard", str(body))
+        doi_filter = body["query"]["bool"]["filter"][0]["bool"]
+        self.assertEqual(doi_filter["minimum_should_match"], 1)
+        self.assertIn(
+            {"prefix": {"doi.keyword": "https://doi.org/10.1590/0034-7167.202578supl101"}},
+            doi_filter["should"],
+        )
+        self.assertIn(
+            {"term": {"doi.keyword": "10.1590/0034-7167.202578supl101"}},
+            doi_filter["should"],
+        )
+
     def _openalex_article(self, openalex_id, language, doi_suffix):
         titles = {
             "en": "Ethical dilemmas in nursing professionals' work",

search_gateway/fixtures/datasources.json

@@ -751,6 +751,21 @@
               "group_order": 0
             }
           },
+          "collection": {
+            "kind": "index",
+            "index_field_name": "oca_data.scielo.collection",
+            "filter": {
+              "size": 100,
+              "order": { "_key": "asc" }
+            },
+            "settings": {
+              "label": "SciELO Collection",
+              "widget": "select",
+              "multiple_selection": true,
+              "group": "coverage",
+              "group_order": 0
+            }
+          },
 
           "publication_year": {
             "kind": "index",
@@ -1248,6 +1263,7 @@
                 { "value": "scope", "label": "Dataset Coverage" },
                 { "value": "source_indexed_in", "label": "Indexed In (OpenAlex)" },
                 { "value": "source_scielo_indexed_in", "label": "Indexed In (SciELO)" },
+                { "value": "collection", "label": "SciELO Collection" },
                 { "value": "document_type", "label": "Document Type" },
                 { "value": "document_language", "label": "Document Language" },
                 { "value": "open_access", "label": "Open Access" },
@@ -1281,6 +1297,7 @@
               "scope",
               "source_indexed_in",
               "source_scielo_indexed_in",
+              "collection",
 
               "publication_year",
               "document_type",
@@ -1366,6 +1383,7 @@
               "scope",
               "source_indexed_in",
               "source_scielo_indexed_in",
+              "collection",
 
               "publication_year",
               "document_type",