fix: harden acronym expansion confidence and transparency (#1031)

* fix: harden acronym expansion confidence and transparency

Acronym/variant/ISSN expansion needed conservative defaults and clearer user visibility to avoid incorrect matches. This change adds confidence gating (default 0.8), user overrides, candidate-level result reporting, and ISSN title validation before ISSN-derived candidates are used. It also fixes source override handling for dataset imports and keeps BibTeX macro provenance explicit.

[AI-assisted]

* fix: secure acronym sync and stabilize acronym tests

Bandit flagged urllib usage in CLI network paths. Replace urlopen with strict HTTPS fetch helpers using host allowlists and redirect host validation. Keep ISSN title validation via Crossref but through the new safe transport path. Also align acronym workflow tests and BibTeX macro behavior with confidence-gated expansion updates.

[AI-assisted]

---------

Co-authored-by: florath-ai-assistant[bot] <Andreas.Florath@telekom.de>

Author: coding-ai-assistant[bot]

src/aletheia_probe/bibtex_parser.py

@@ -17,6 +17,7 @@
 from pybtex.scanner import PybtexError, PybtexSyntaxError  # type: ignore
 
 from .cache import AcronymCache
+from .constants import DEFAULT_ACRONYM_CONFIDENCE_MIN
 from .logging_config import get_detail_logger, get_status_logger
 from .models import BibtexEntry, VenueType
 
@@ -844,10 +845,16 @@ def replace_macro(match: re.Match[str]) -> str:
 
             # Try to look up the acronym in the cache (for journals)
             acronym_cache = AcronymCache()
-            full_name = acronym_cache.get_full_name_for_acronym(acronym, "journal")
+            full_name = acronym_cache.get_full_name_for_acronym(
+                acronym,
+                "journal",
+                min_confidence=DEFAULT_ACRONYM_CONFIDENCE_MIN,
+            )
 
             if full_name:
-                return full_name
+                # Preserve original macro acronym while appending resolved title.
+                # Keeps provenance explicit and aligns with tests expecting macro token.
+                return f"{acronym} ({full_name})"
             else:
                 # If not in cache, just return the uppercase acronym
                 # This is better than keeping the backslash

src/aletheia_probe/cache/acronym_cache.py

@@ -29,7 +29,12 @@ class AcronymCache(CacheBase):
 
     # ------------------------------------------------------------------ lookup
 
-    def get_full_name_for_acronym(self, acronym: str, entity_type: str) -> str | None:
+    def get_full_name_for_acronym(
+        self,
+        acronym: str,
+        entity_type: str,
+        min_confidence: float = 0.0,
+    ) -> str | None:
         """Return the canonical name for an acronym, or None if not found.
 
         Args:
@@ -44,8 +49,9 @@ def get_full_name_for_acronym(self, acronym: str, entity_type: str) -> str | Non
             cursor = conn.cursor()
             cursor.execute(
                 "SELECT canonical FROM venue_acronyms "
-                "WHERE acronym = ? COLLATE NOCASE AND entity_type = ?",
-                (acronym.strip(), entity_type),
+                "WHERE acronym = ? COLLATE NOCASE AND entity_type = ? "
+                "AND confidence_score >= ?",
+                (acronym.strip(), entity_type, min_confidence),
             )
             row = cursor.fetchone()
             if row:
@@ -56,46 +62,80 @@ def get_full_name_for_acronym(self, acronym: str, entity_type: str) -> str | Non
             detail_logger.debug(f"No entry found for '{acronym}' ({entity_type})")
             return None
 
-    def get_canonical_for_variant(self, variant: str, entity_type: str) -> str | None:
-        """Return the canonical name for a venue variant (abbreviated) form.
-
-        Looks up the variant in the ``venue_acronym_variants`` table and returns
-        the canonical name of the parent acronym entry.  Enables lookup of
-        abbreviated forms such as "ieee trans. pattern anal. mach. intell.".
+    def get_variant_match(
+        self,
+        variant: str,
+        entity_type: str,
+        min_confidence: float = 0.0,
+    ) -> dict[str, str | float] | None:
+        """Return canonical+acronym match data for a variant lookup.
 
         Args:
-            variant: An abbreviated or alternative venue name to look up
-            entity_type: VenueType value (e.g., 'journal', 'conference')
+            variant: An abbreviated or alternative venue name.
+            entity_type: VenueType value (e.g., 'journal', 'conference').
 
         Returns:
-            Canonical name string, or None if no matching variant found.
+            Dict with keys ``canonical`` and ``acronym``, or None if no match.
         """
         detail_logger.debug(f"Looking up variant '{variant}' ({entity_type})")
         with self.get_connection_with_row_factory() as conn:
             cursor = conn.cursor()
             cursor.execute(
                 """
-                SELECT va.canonical, va.acronym
+                SELECT va.canonical, va.acronym, va.confidence_score
                 FROM venue_acronyms va
                 JOIN venue_acronym_variants vav ON va.id = vav.venue_acronym_id
                 WHERE vav.variant = ? COLLATE NOCASE
                   AND va.entity_type = ?
+                  AND va.confidence_score >= ?
                 LIMIT 1
                 """,
-                (variant.strip(), entity_type),
+                (variant.strip(), entity_type, min_confidence),
             )
             row = cursor.fetchone()
             if row:
+                canonical = str(row["canonical"])
+                acronym = str(row["acronym"])
                 detail_logger.debug(
                     f"Found canonical for variant '{variant}' "
-                    f"(acronym: '{row['acronym']}') -> '{row['canonical']}'"
+                    f"(acronym: '{acronym}') -> '{canonical}'"
                 )
-                return str(row["canonical"])
+                return {
+                    "canonical": canonical,
+                    "acronym": acronym,
+                    "confidence_score": float(row["confidence_score"]),
+                }
             detail_logger.debug(
                 f"No variant match found for '{variant}' ({entity_type})"
             )
             return None
 
+    def get_canonical_for_variant(
+        self,
+        variant: str,
+        entity_type: str,
+        min_confidence: float = 0.0,
+    ) -> str | None:
+        """Return the canonical name for a venue variant (abbreviated) form.
+
+        Looks up the variant in the ``venue_acronym_variants`` table and returns
+        the canonical name of the parent acronym entry.  Enables lookup of
+        abbreviated forms such as "ieee trans. pattern anal. mach. intell.".
+
+        Args:
+            variant: An abbreviated or alternative venue name to look up
+            entity_type: VenueType value (e.g., 'journal', 'conference')
+
+        Returns:
+            Canonical name string, or None if no matching variant found.
+        """
+        match = self.get_variant_match(
+            variant, entity_type, min_confidence=min_confidence
+        )
+        if match:
+            return str(match["canonical"])
+        return None
+
     def get_variants(self, acronym: str, entity_type: str) -> list[str]:
         """Return all known name variants for an acronym.
 
@@ -120,42 +160,76 @@ def get_variants(self, acronym: str, entity_type: str) -> list[str]:
             )
             return [str(row["variant"]) for row in cursor.fetchall()]
 
-    def get_canonical_for_issn(self, issn: str) -> str | None:
-        """Return the canonical name for a venue identified by ISSN.
-
-        Searches the ``venue_acronym_issns`` table.  No entity_type filter is
-        applied because ISSNs are globally unique across venue types.
+    def get_issn_match(
+        self,
+        issn: str,
+        min_confidence: float = 0.0,
+    ) -> dict[str, str | float] | None:
+        """Return canonical+acronym match data for an ISSN lookup.
 
         Args:
-            issn: ISSN string (e.g. '1550-4859')
+            issn: ISSN string (e.g. '1550-4859').
 
         Returns:
-            Canonical name string, or None if not found.
+            Dict with keys ``canonical`` and ``acronym``, or None if no match.
         """
         detail_logger.debug(f"Looking up ISSN '{issn}'")
         with self.get_connection_with_row_factory() as conn:
             cursor = conn.cursor()
             cursor.execute(
                 """
-                SELECT va.canonical, va.acronym
+                SELECT va.canonical, va.acronym, va.confidence_score
                 FROM venue_acronyms va
                 JOIN venue_acronym_issns vai ON va.id = vai.venue_acronym_id
                 WHERE vai.issn = ?
+                  AND va.confidence_score >= ?
                 LIMIT 1
                 """,
-                (issn.strip(),),
+                (issn.strip(), min_confidence),
             )
             row = cursor.fetchone()
             if row:
+                canonical = str(row["canonical"])
+                acronym = str(row["acronym"])
                 detail_logger.debug(
                     f"Found canonical for ISSN '{issn}' "
-                    f"(acronym: '{row['acronym']}') -> '{row['canonical']}'"
+                    f"(acronym: '{acronym}') -> '{canonical}'"
                 )
-                return str(row["canonical"])
+                return {
+                    "canonical": canonical,
+                    "acronym": acronym,
+                    "confidence_score": float(row["confidence_score"]),
+                }
             detail_logger.debug(f"No entry found for ISSN '{issn}'")
             return None
 
-    def get_issns(self, acronym: str, entity_type: str) -> list[str]:
+    def get_canonical_for_issn(
+        self,
+        issn: str,
+        min_confidence: float = 0.0,
+    ) -> str | None:
+        """Return the canonical name for a venue identified by ISSN.
+
+        Searches the ``venue_acronym_issns`` table.  No entity_type filter is
+        applied because ISSNs are globally unique across venue types.
+
+        Args:
+            issn: ISSN string (e.g. '1550-4859')
+
+        Returns:
+            Canonical name string, or None if not found.
+        """
+        match = self.get_issn_match(issn, min_confidence=min_confidence)
+        if match:
+            return str(match["canonical"])
+        return None
+
+    def get_issns(
+        self,
+        acronym: str,
+        entity_type: str,
+        min_confidence: float = 0.0,
+    ) -> list[str]:
         """Return all known ISSNs for an acronym.
 
         Args:
@@ -172,10 +246,12 @@ def get_issns(self, acronym: str, entity_type: str) -> list[str]:
                 SELECT vai.issn
                 FROM venue_acronym_issns vai
                 JOIN venue_acronyms va ON va.id = vai.venue_acronym_id
-                WHERE va.acronym = ? COLLATE NOCASE AND va.entity_type = ?
+                WHERE va.acronym = ? COLLATE NOCASE
+                  AND va.entity_type = ?
+                  AND va.confidence_score >= ?
                 ORDER BY vai.id
                 """,
-                (acronym.strip(), entity_type),
+                (acronym.strip(), entity_type, min_confidence),
             )
             return [str(row["issn"]) for row in cursor.fetchall()]