Merge pull request #4787 from acl-org/python-normalize-and-latex-import

Implement conversion from LaTeX to our Markup XML, reimplementing `bin/latex_to_unicode.py`.

Author: mbollmann

python/CHANGELOG.md

@@ -1,5 +1,12 @@
 # Changelog
 
+## Unreleased
+
+### Added
+
+- MarkupText can now be instantiated from strings (potentially) containing LaTeX markup.
+  - This reimplements functionality used at ingestion time previously found in `bin/latex_to_unicode.py`.
+
 ## [0.5.2] — 2025-05-16
 
 This release adds support for Python 3.13 and initial functionality for creating new proceedings.

python/acl_anthology/exceptions.py

@@ -1,4 +1,4 @@
-# Copyright 2023-2024 Marcel Bollmann <[email protected]>
+# Copyright 2023-2025 Marcel Bollmann <[email protected]>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -24,6 +24,8 @@
 if sys.version_info >= (3, 11):
 
     class AnthologyException(Exception):
+        """Base class from which all other exceptions defined here inherit."""
+
         def __init__(self, msg: str):
             super().__init__(msg)
 

python/acl_anthology/text/markuptext.py

@@ -1,4 +1,4 @@
-# Copyright 2019-2024 Marcel Bollmann <[email protected]>
+# Copyright 2019-2025 Marcel Bollmann <[email protected]>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -24,8 +24,10 @@
 from xml.sax.saxutils import escape as xml_escape
 
 from ..utils import (
+    clean_unicode,
     latex_encode,
     latex_convert_quotes,
+    parse_latex_to_xml,
     remove_extra_whitespace,
     stringify_children,
 )
@@ -171,16 +173,51 @@ def as_xml(self) -> str:
         return self._xml
 
     @classmethod
-    def from_string(cls, text: str) -> MarkupText:
+    def from_string(cls, text: str, clean: bool = True) -> MarkupText:
         """
         Arguments:
             text: A simple text string without any markup.
+            clean: If True, applies the Anthology's [Unicode normalization][acl_anthology.utils.text.clean_unicode].
 
         Returns:
             Instantiated MarkupText object corresponding to the string.
         """
+        if clean:
+            return cls(clean_unicode(text))
         return cls(text)
 
+    @classmethod
+    def from_latex(cls, text: str, clean: bool = True) -> MarkupText:
+        """
+        Arguments:
+            text: A text string potentially containing LaTeX markup.
+            clean: If True, applies the Anthology's [Unicode normalization][acl_anthology.utils.text.clean_unicode].
+
+        Returns:
+            Instantiated MarkupText object corresponding to the string.
+        """
+        if clean:
+            text = clean_unicode(text)
+        element = parse_latex_to_xml(text, use_heuristics=False)
+        return cls.from_xml(element)
+
+    @classmethod
+    def from_latex_maybe(cls, text: str, clean: bool = True) -> MarkupText:
+        """
+        Like `from_latex()`, but can be used if it is unclear if the string is plain text or LaTeX.  Will prevent percentage signs being interpreted as LaTeX comments, and apply a heuristic to decide if a tilde is literal or a non-breaking space.
+
+        Arguments:
+            text: A text string potentially in plain text or LaTeX format.
+            clean: If True, applies the Anthology's [Unicode normalization][acl_anthology.utils.text.clean_unicode].
+
+        Returns:
+            Instantiated MarkupText object corresponding to the string.
+        """
+        if clean:
+            text = clean_unicode(text)
+        element = parse_latex_to_xml(text, use_heuristics=True)
+        return cls.from_xml(element)
+
     @classmethod
     def from_xml(cls, element: etree._Element) -> MarkupText:
         """

python/acl_anthology/utils/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2023-2024 Marcel Bollmann <[email protected]>
+# Copyright 2023-2025 Marcel Bollmann <[email protected]>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,22 +15,29 @@
 from .citation import citeproc_render_html
 from .git import clone_or_pull_from_repo
 from .ids import build_id, parse_id, AnthologyID
-from .latex import latex_encode, latex_convert_quotes, make_bibtex_entry
+from .latex import (
+    latex_encode,
+    latex_convert_quotes,
+    make_bibtex_entry,
+    parse_latex_to_xml,
+)
 from .logging import setup_rich_logging, get_logger
-from .text import remove_extra_whitespace
+from .text import clean_unicode, remove_extra_whitespace
 from .xml import stringify_children
 
 
 __all__ = [
     "AnthologyID",
     "build_id",
     "citeproc_render_html",
+    "clean_unicode",
     "clone_or_pull_from_repo",
     "get_logger",
     "latex_encode",
     "latex_convert_quotes",
     "make_bibtex_entry",
     "parse_id",
+    "parse_latex_to_xml",
     "remove_extra_whitespace",
     "setup_rich_logging",
     "stringify_children",