Merge pull request #688 from pitangainnovare/reduz-escopo-de-busca-openalex

Reduz escopo de busca openalex em dedup com scielo

Author: pitangainnovare

etl/deduplicator/openalex.py

@@ -47,6 +47,12 @@ def find_matches(
 
         primary = select_primary_scielo_doc(scielo_group)
         rules = self.rules
+        if not self._can_search_openalex(primary):
+            logger.debug(
+                "Skipping OpenAlex match lookup for SciELO doc outside configured query scope"
+            )
+            return []
+
         matches = []
 
         for strategy in rules["openalex_match_strategies"]:
@@ -119,13 +125,71 @@ def _deduplicate_openalex_matches(self, matches: list) -> list:
                 deduped.append(match)
         return deduped
 
+    def _can_search_openalex(self, scielo_doc: dict) -> bool:
+        year = self._publication_year(scielo_doc)
+        if year is None:
+            return False
+
+        query_rules = self.rules.get("openalex_query") or {}
+        min_year = query_rules.get("publication_year_min")
+        max_year = query_rules.get("publication_year_max")
+        if min_year is None and max_year is None:
+            return True
+
+        tolerance = self._year_tolerance()
+        lower = year - tolerance
+        upper = year + tolerance
+
+        try:
+            if min_year is not None and upper < int(min_year):
+                return False
+            if max_year is not None and lower > int(max_year):
+                return False
+        except (TypeError, ValueError):
+            logger.warning("Invalid OpenAlex publication year query bounds: %s", query_rules)
+            return True
+
+        return True
+
+    def _publication_year(self, doc: dict) -> int | None:
+        try:
+            return int(doc.get("publication_year"))
+        except (TypeError, ValueError):
+            return None
+
+    def _year_tolerance(self) -> int:
+        validation_rules = self.rules.get("openalex_validation") or {}
+        try:
+            return int(validation_rules.get("year_tolerance", 0) or 0)
+        except (TypeError, ValueError):
+            return 0
+
+    def _apply_openalex_query_constraints(
+        self,
+        query: dict[str, Any],
+        scielo_doc: dict[str, Any],
+    ) -> dict[str, Any]:
+        bool_query = query.setdefault("bool", {})
+        query_rules = self.rules.get("openalex_query") or {}
+
+        if query_rules.get("exclude_is_xpac"):
+            bool_query.setdefault("must_not", []).append({"term": {"is_xpac": True}})
+
+        year = self._publication_year(scielo_doc)
+        if year is not None:
+            tolerance = self._year_tolerance()
+            bool_query.setdefault("filter", []).append(
+                {"range": {"publication_year": {"gte": year - tolerance, "lte": year + tolerance}}}
+            )
+
+        return query
+
     def _search_openalex_by_doi(
         self,
         doi: str,
         scielo_doc: Dict[str, Any],
         size: int = 10,
     ) -> List[Dict[str, Any]]:
-        year = scielo_doc.get("publication_year")
         normalized_doi = normalize_doi(doi)
         if not normalized_doi:
             logger.warning("Invalid DOI after normalization: %s", doi)
@@ -143,14 +207,7 @@ def _search_openalex_by_doi(
                 ]
             }
         }
-        if year is not None:
-            try:
-                year = int(year)
-                query["bool"]["filter"].append(
-                    {"range": {"publication_year": {"gte": year - 1, "lte": year + 1}}}
-                )
-            except (ValueError, TypeError):
-                pass
+        self._apply_openalex_query_constraints(query, scielo_doc)
 
         try:
             response = self.client.client.search(
@@ -185,7 +242,6 @@ def _search_openalex_by_isbn(
         scielo_doc: Dict[str, Any],
         size: int = 10,
     ) -> List[Dict[str, Any]]:
-        year = scielo_doc.get("publication_year")
         query = {
             "bool": {
                 "should": [
@@ -201,15 +257,7 @@ def _search_openalex_by_isbn(
                 "minimum_should_match": 1,
             }
         }
-
-        if year is not None:
-            try:
-                year = int(year)
-                query["bool"]["filter"] = [
-                    {"range": {"publication_year": {"gte": year - 1, "lte": year + 1}}}
-                ]
-            except (ValueError, TypeError):
-                pass
+        self._apply_openalex_query_constraints(query, scielo_doc)
 
         try:
             response = self.client.client.search(
@@ -227,7 +275,6 @@ def _search_openalex_by_title_year(
         size: int = 10,
     ) -> List[Dict[str, Any]]:
         title = scielo_doc.get("title", "")
-        year = scielo_doc.get("publication_year")
         issns = scielo_doc.get("source_issns", [])
         if not title:
             return []
@@ -247,16 +294,7 @@ def _search_openalex_by_title_year(
                 ]
             }
         }
-
-        if year:
-            try:
-                year_int = int(year)
-            except (TypeError, ValueError):
-                logger.warning("Invalid year value: %s, skipping range filter", year)
-                return []
-            query["bool"]["filter"] = [
-                {"range": {"publication_year": {"gte": year_int - 1, "lte": year_int + 1}}}
-            ]
+        self._apply_openalex_query_constraints(query, scielo_doc)
 
         if issns:
             query["bool"]["should"] = [{"terms": {"source.issns": issns}}]

etl/fixtures/pipeline_configs.json

@@ -13,6 +13,11 @@
             "rules": {
                 "scielo_dedup_strategies": ["doi", "pid", "fuzzy"],
                 "openalex_match_strategies": ["doi", "isbn", "title"],
+                "openalex_query": {
+                    "exclude_is_xpac": true,
+                    "publication_year_min": 2019,
+                    "publication_year_max": 2025
+                },
                 "doi_requires_title_overlap": true,
                 "pid_requires_year_match": true,
                 "pid_requires_source_match": true,

etl/models.py

@@ -54,6 +54,12 @@
     "isbn_title_threshold": 0.80,
 }
 
+DEFAULT_OPENALEX_QUERY_RULES = {
+    "exclude_is_xpac": False,
+    "publication_year_min": None,
+    "publication_year_max": None,
+}
+
 
 class EtlStatus(models.TextChoices):
     PENDING = "pending", "Pending"
@@ -240,6 +246,7 @@ def to_rules(self) -> dict:
         self.clean()
         rules = self.rules or {}
         oa_val = rules.get("openalex_validation", {})
+        oa_query = rules.get("openalex_query", {})
         return {
             "document_type": normalize_document_type_for_etl(self.default_document_type),
             "scielo_dedup_strategies": list(rules.get("scielo_dedup_strategies", [])),
@@ -255,6 +262,10 @@ def to_rules(self) -> dict:
                 **DEFAULT_OPENALEX_VALIDATION_RULES,
                 **oa_val,
             },
+            "openalex_query": {
+                **DEFAULT_OPENALEX_QUERY_RULES,
+                **oa_query,
+            },
         }
 
 

etl/tests/pipeline/test_pipeline_dedup.py

@@ -1,12 +1,11 @@
 from unittest.mock import Mock
 
-from django.test import TestCase
-
 from etl.deduplicator.helpers import can_compare, rules_for_pair
 from etl.deduplicator.openalex import OpenAlexMatcher
 from etl.deduplicator.scielo import SciELODeduplicator
 from etl.models import EtlPipelineConfig
 from etl.pipeline import OpenSearchETLPipeline
+from etl.tests.base import EtlTestCase
 
 
 def make_deduplicator(document_type):
@@ -19,7 +18,7 @@ def make_matcher(document_type):
     return matcher
 
 
-class DocumentRulesTests(TestCase):
+class DocumentRulesTests(EtlTestCase):
     def test_can_compare_rejects_doc_type_outside_pipeline_config_rules(self):
         rules = EtlPipelineConfig.objects.get_enabled_by_name("book").to_rules()
 
@@ -229,6 +228,14 @@ def test_openalex_doi_search_uses_exact_or_prefix_queries(self):
         self.assertNotIn("wildcard", str(body))
         doi_filter = body["query"]["bool"]["filter"][0]["bool"]
         self.assertEqual(doi_filter["minimum_should_match"], 1)
+        self.assertEqual(
+            body["query"]["bool"]["must_not"],
+            [{"term": {"is_xpac": True}}],
+        )
+        self.assertIn(
+            {"range": {"publication_year": {"gte": 2024, "lte": 2026}}},
+            body["query"]["bool"]["filter"],
+        )
         self.assertIn(
             {"prefix": {"doi.keyword": "https://doi.org/10.1590/0034-7167.202578supl101"}},
             doi_filter["should"],
@@ -238,6 +245,62 @@ def test_openalex_doi_search_uses_exact_or_prefix_queries(self):
             doi_filter["should"],
         )
 
+    def test_openalex_match_skips_year_outside_configured_raw_range(self):
+        matcher = make_matcher("article")
+        matcher.input_openalex_index = "raw_openalex_works"
+        matcher.client = Mock()
+
+        matches = matcher.find_matches(
+            [
+                {
+                    "type": "article",
+                    "publication_year": 2017,
+                    "ids": {"doi": "10.1590/0034-7167.202578SUPL101"},
+                }
+            ],
+            max_candidates=3,
+        )
+
+        self.assertEqual(matches, [])
+        matcher.client.client.search.assert_not_called()
+
+    def test_openalex_match_keeps_year_adjacent_to_configured_raw_range(self):
+        matcher = make_matcher("article")
+        matcher.input_openalex_index = "raw_openalex_works"
+        matcher.client = Mock()
+        matcher.client.client.search.return_value = {"hits": {"hits": []}}
+
+        matcher.find_matches(
+            [
+                {
+                    "type": "article",
+                    "publication_year": 2018,
+                    "ids": {"doi": "10.1590/0034-7167.202578SUPL101"},
+                }
+            ],
+            max_candidates=3,
+        )
+
+        matcher.client.client.search.assert_called_once()
+
+    def test_openalex_match_skips_missing_scielo_publication_year(self):
+        matcher = make_matcher("article")
+        matcher.input_openalex_index = "raw_openalex_works"
+        matcher.client = Mock()
+
+        matches = matcher.find_matches(
+            [
+                {
+                    "type": "article",
+                    "ids": {"doi": "10.1590/0034-7167.202578SUPL101"},
+                }
+            ],
+            max_candidates=3,
+        )
+
+        self.assertEqual(matches, [])
+        matcher.client.client.search.assert_not_called()
+
     def _openalex_article(self, openalex_id, language, doi_suffix):
         titles = {
             "en": "Ethical dilemmas in nursing professionals' work",

search_gateway/tests/test_transforms.py

@@ -5,7 +5,7 @@
 
 class ScopeDisplayTransformTests(SimpleTestCase):
     def test_openalex_works_label(self):
-        self.assertEqual(apply_display_transform("scope", "openalex_works"), "OpenAlex")
+        self.assertEqual(apply_display_transform("scope", "openalex"), "OpenAlex")
 
     def test_scielo_label(self):
         self.assertEqual(apply_display_transform("scope", "scielo"), "SciELO")