feat(parser,backend,frontend): enable extended taxonomies (#429)

* enable extension of taxonomies

* fix edge case where node merged with external node

* handle external nodes on frontend

* chore: resync-pr

* resolve comments

* reintroduce deleted test

* move the external taxonomies parsing logic to taxonomy_parser.py

---------

Co-authored-by: Charles Perier <[email protected]>
Co-authored-by: Alex Garel <[email protected]>

Author: 3 people

backend/editor/entries.py

@@ -2,6 +2,7 @@
 Database helper functions for API
 """
 
+import asyncio
 import logging
 import re
 import shutil
@@ -34,6 +35,7 @@
 )
 from .models.node_models import EntryNodeCreate
 from .models.project_models import ProjectCreate, ProjectEdit, ProjectStatus
+from .settings import EXTERNAL_TAXONOMIES
 
 log = logging.getLogger(__name__)
 
@@ -50,9 +52,8 @@ def __init__(self, branch_name, taxonomy_name):
         self.branch_name = branch_name
         self.project_name = "p_" + taxonomy_name + "_" + branch_name
 
-    @property
-    def taxonomy_path_in_repository(self):
-        return utils.taxonomy_path_in_repository(self.taxonomy_name)
+    def taxonomy_path_in_repository(self, taxonomy_name):
+        return utils.taxonomy_path_in_repository(taxonomy_name)
 
     def get_label(self, id):
         """
@@ -86,29 +87,34 @@ async def get_local_taxonomy_file(self, tmpdir: str, uploadfile: UploadFile):
             await run_in_threadpool(shutil.copyfileobj, uploadfile.file, f)
         return filepath
 
-    async def get_github_taxonomy_file(self, tmpdir: str):
+    async def get_github_taxonomy_file(self, tmpdir: str, taxonomy_name: str):
         async with TransactionCtx():
-            filepath = f"{tmpdir}/{self.taxonomy_name}.txt"
+            filepath = f"{tmpdir}/{taxonomy_name}.txt"
+            path_in_repository = self.taxonomy_path_in_repository(taxonomy_name)
             target_url = (
                 f"https://raw.githubusercontent.com/{settings.repo_uri}"
-                f"/main/{self.taxonomy_path_in_repository}"
+                f"/main/{path_in_repository}"
             )
             try:
+                # get taxonomy file
                 await run_in_threadpool(urllib.request.urlretrieve, target_url, filepath)
-                github_object = GithubOperations(self.taxonomy_name, self.branch_name)
-                commit_sha = (await github_object.get_branch("main")).commit.sha
-                file_sha = await github_object.get_file_sha()
-                await edit_project(
-                    self.project_name,
-                    ProjectEdit(
-                        github_checkout_commit_sha=commit_sha, github_file_latest_sha=file_sha
-                    ),
-                )
+                if taxonomy_name == self.taxonomy_name:
+                    # this is the taxonomy we want to edit
+                    # track the current commit to know where to start the PR from
+                    github_object = GithubOperations(self.taxonomy_name, self.branch_name)
+                    commit_sha = (await github_object.get_branch("main")).commit.sha
+                    file_sha = await github_object.get_file_sha()
+                    await edit_project(
+                        self.project_name,
+                        ProjectEdit(
+                            github_checkout_commit_sha=commit_sha, github_file_latest_sha=file_sha
+                        ),
+                    )
                 return filepath
             except Exception as e:
                 raise TaxonomyImportError() from e
 
-    def parse_taxonomy(self, filepath: str):
+    def parse_taxonomy(self, main_filepath: str, other_filepaths: list[str] | None = None):
         """
         Helper function to call the Open Food Facts Python Taxonomy Parser
         """
@@ -117,7 +123,7 @@ def parse_taxonomy(self, filepath: str):
             parser_object = parser.Parser(session)
             try:
                 # Parse taxonomy with given file name and branch name
-                parser_object(filepath, self.branch_name, self.taxonomy_name)
+                parser_object(main_filepath, other_filepaths, self.branch_name, self.taxonomy_name)
             except Exception as e:
                 # outer exception handler will put project status to FAILED
                 raise TaxonomyParsingError() from e
@@ -126,11 +132,14 @@ async def get_and_parse_taxonomy(self, uploadfile: UploadFile | None = None):
         try:
             with tempfile.TemporaryDirectory(prefix="taxonomy-") as tmpdir:
                 filepath = await (
-                    self.get_github_taxonomy_file(tmpdir)
+                    self.get_github_taxonomy_file(tmpdir, self.taxonomy_name)
                     if uploadfile is None
                     else self.get_local_taxonomy_file(tmpdir, uploadfile)
                 )
-                await run_in_threadpool(self.parse_taxonomy, filepath)
+                other_filepaths = None
+                if self.taxonomy_name in EXTERNAL_TAXONOMIES:
+                    other_filepaths = await self.fetch_external_taxonomy_files(tmpdir)
+                await run_in_threadpool(self.parse_taxonomy, filepath, other_filepaths)
                 async with TransactionCtx():
                     error_node = await get_error_node(self.project_name)
                     errors_count = len(error_node.errors) if error_node else 0
@@ -149,6 +158,25 @@ async def get_and_parse_taxonomy(self, uploadfile: UploadFile | None = None):
             log.exception(e)
             raise e
 
+    async def fetch_external_taxonomy_files(self, tmpdir: str) -> list[str]:
+        """
+        Helper function to fetch external taxonomies concurrently from Github
+        """
+        external_taxonomy_filepaths = []
+        tasks = []
+
+        # Create tasks for each external taxonomy and store them in a list
+        for external_taxonomy in EXTERNAL_TAXONOMIES[self.taxonomy_name]:
+            task = asyncio.create_task(self.get_github_taxonomy_file(tmpdir, external_taxonomy))
+            tasks.append(task)
+
+        # Wait for all tasks to complete concurrently
+        for task in tasks:
+            external_filepath = await task
+            external_taxonomy_filepaths.append(external_filepath)
+
+        return external_taxonomy_filepaths
+
     async def import_taxonomy(
         self,
         description: str,
@@ -689,9 +717,8 @@ async def full_text_search(self, text):
                     where score_ > 0
                     return node, score_ as score
             }
-            with node.id as node, score
-            RETURN node, sum(score) as score
-
+            WITH node.id AS node_id, node.is_external AS is_external, score
+            RETURN {id: node_id, is_external: is_external} AS node, sum(score) AS score
             ORDER BY score DESC
         """
         _result = await get_current_transaction().run(query, params)

backend/editor/settings.py

@@ -6,3 +6,14 @@
 uri = os.environ.get("NEO4J_URI", "bolt://localhost:7687")
 access_token = os.environ.get("GITHUB_PAT")
 repo_uri = os.environ.get("REPO_URI", "openfoodfacts/openfoodfacts-server")
+
+EXTERNAL_TAXONOMIES = {
+    "food_ingredients": [
+        "additives_classes",
+        "additives",
+        "minerals",
+        "vitamins",
+        "nucleotides",
+        "other_nutritional_substances",
+    ],
+}

parser/openfoodfacts_taxonomy_parser/parser/parser.py

@@ -28,7 +28,7 @@ def _create_other_node(self, tx: Transaction, node_data: NodeData, project_label
         elif node_data.get_node_type() == NodeType.STOPWORDS:
             type_label = "STOPWORDS"
         else:
-            raise ValueError(f"ENTRY nodes should not be passed to this function")
+            raise ValueError("ENTRY nodes should not be passed to this function")
 
         node_tags_queries = [f"{key} : ${key}" for key in node_data.tags]
 
@@ -70,7 +70,7 @@ def _create_entry_nodes(self, entry_nodes: list[NodeData], project_label: str):
 
         for entry_node in entry_nodes:
             if entry_node.get_node_type() != NodeType.ENTRY:
-                raise ValueError(f"Only ENTRY nodes should be passed to this function")
+                raise ValueError("Only ENTRY nodes should be passed to this function")
             seen_properties_and_tags_and_comments.update(entry_node.tags)
             seen_properties_and_tags_and_comments.update(entry_node.properties)
             seen_properties_and_tags_and_comments.update(entry_node.comments)
@@ -79,11 +79,13 @@ def _create_entry_nodes(self, entry_nodes: list[NodeData], project_label: str):
             f"{key} : entry_node.{key}" for key in seen_properties_and_tags_and_comments
         ]
 
-        base_properties_query = f"""
+        base_properties_query = """
             id: entry_node.id,
             preceding_lines: entry_node.preceding_lines,
             src_position: entry_node.src_position,
-            main_language: entry_node.main_language
+            main_language: entry_node.main_language,
+            is_external: entry_node.is_external,
+            original_taxonomy: entry_node.original_taxonomy
         """
 
         properties_query = ",\n".join([base_properties_query, *additional_properties_queries])
@@ -234,14 +236,23 @@ def _write_to_database(self, taxonomy: Taxonomy, taxonomy_name: str, branch_name
         self._create_child_links(taxonomy.child_links, project_label)
         self._create_previous_links(taxonomy.previous_links, project_label)
 
-    def __call__(self, filename: str, branch_name: str, taxonomy_name: str):
+    def __call__(
+        self,
+        main_filename: str,
+        external_filenames: list[str] | None,
+        branch_name: str,
+        taxonomy_name: str,
+    ):
         """Process the file"""
         start_time = timeit.default_timer()
 
         branch_name = normalize_text(branch_name, char="_")
         taxonomy_parser = TaxonomyParser()
         try:
-            taxonomy = taxonomy_parser.parse_file(filename, self.parser_logger)
+            taxonomy = taxonomy_parser.parse_file(
+                main_filename, external_filenames, self.parser_logger
+            )
+
             self._write_to_database(taxonomy, taxonomy_name, branch_name)
 
             self.parser_logger.info(

parser/openfoodfacts_taxonomy_parser/parser/taxonomy_parser.py

@@ -35,14 +35,18 @@ class NodeData:
     # to keep track of comments just above the current line
     # during parsing of an entry, to be able to add them
     # to the right property or tag when possible
-    comments_stack: list[(int, str)] = field(default_factory=list)
+    comments_stack: list[tuple[int, str]] = field(default_factory=list)
+    is_external: bool = False  # True if the node comes from another taxonomy
+    original_taxonomy: str | None = None  # the name of the taxonomy the node comes from
 
     def to_dict(self):
         return {
             "id": self.id,
             "main_language": self.main_language,
             "preceding_lines": self.preceding_lines,
             "src_position": self.src_position,
+            "is_external": self.is_external,
+            "original_taxonomy": self.original_taxonomy,
             **self.properties,
             **self.tags,
             **self.comments,
@@ -464,6 +468,14 @@ def _merge_duplicate_entry_nodes(self, entry_nodes: list[NodeData]) -> list[Node
         for node in entry_nodes:
             if node.id in ids_to_nodes:
                 first_node = ids_to_nodes[node.id]
+                if first_node.is_external:
+                    # we don't want to merge a node with an external node;
+                    # the external node gets a new id with its original taxonomy name
+                    # and the new one becomes the new "first node"
+                    first_node.id += f"@{first_node.original_taxonomy}"
+                    unique_entry_nodes.append(node)
+                    ids_to_nodes[node.id] = node
+                    continue
                 for key, value in node.tags.items():
                     if not key.startswith("tags_ids_"):
                         # union of the tags
@@ -496,11 +508,25 @@ def _merge_duplicate_entry_nodes(self, entry_nodes: list[NodeData]) -> list[Node
                 ids_to_nodes[node.id] = node
         return unique_entry_nodes
 
-    def _create_taxonomy(self, filename: str) -> Taxonomy:
+    def _create_taxonomy(
+        self, filename: str, external_filenames: list[str] | None = None
+    ) -> Taxonomy:
         """Create the taxonomy from the file"""
+        # parse external taxonomies if any, and add their entry nodes to the main taxonomy
+        external_entry_nodes = []
+        for external_filename in external_filenames or []:
+            external_taxonomy_parser = TaxonomyParser()
+            external_taxonomy = external_taxonomy_parser.parse_file(
+                external_filename, None, self.parser_logger
+            )
+            external_entry_nodes.extend(external_taxonomy.entry_nodes)
+        for node in external_entry_nodes:
+            node.is_external = True
+
         self.parser_logger.info(f"Parsing {filename}")
         harvested_header_data, entries_start_line = self._header_harvest(filename)
         entry_nodes: list[NodeData] = []
+        entry_nodes.extend(external_entry_nodes)
         other_nodes = [
             NodeData(id="__header__", preceding_lines=harvested_header_data, src_position=1)
         ]
@@ -509,6 +535,7 @@ def _create_taxonomy(self, filename: str) -> Taxonomy:
         harvested_data = self._harvest_entries(filename, entries_start_line)
         for entry in harvested_data:
             if entry.get_node_type() == NodeType.ENTRY:
+                entry.original_taxonomy = filename.split("/")[-1]
                 entry_nodes.append(entry)
             else:
                 other_nodes.append(entry)
@@ -534,13 +561,18 @@ def _create_taxonomy(self, filename: str) -> Taxonomy:
             child_links=child_links,
         )
 
-    def parse_file(self, filename: str, logger: ParserConsoleLogger | None = None) -> Taxonomy:
+    def parse_file(
+        self,
+        filename: str,
+        external_filenames: list[str] | None = None,
+        logger: ParserConsoleLogger | None = None,
+    ) -> Taxonomy:
         if logger:
             self.parser_logger = logger
         """Process the file into a Taxonomy object"""
         start_time = timeit.default_timer()
         filename = normalize_filename(filename)
-        taxonomy = self._create_taxonomy(filename)
+        taxonomy = self._create_taxonomy(filename, external_filenames)
         self.parser_logger.info(f"Parsing done in {timeit.default_timer() - start_time} seconds.")
         self.parser_logger.info(
             f"Found {len(taxonomy.entry_nodes) + len(taxonomy.other_nodes)} nodes"

parser/tests/data/test.txt

@@ -6,6 +6,7 @@ synonyms:en:passion fruit, passionfruit
 
 synonyms:fr:fruit de la passion, maracuja, passion
 
+<en:milk
 en:yogurts, yoghurts
 fr:yaourts, yoghourts, yogourts
 

parser/tests/data/test_external1.txt

@@ -0,0 +1,18 @@
+# External taxonomy 1
+
+synonyms:en:fiber, fibre
+
+stopwords:fr:aux, au, de, le, du, la, a, et
+
+# comment
+en:milk
+fr:lait
+
+en:honey
+fr:miel
+
+<en:milk
+en:oat milk
+
+<en:milk
+en:almond milk

parser/tests/data/test_external2.txt

@@ -0,0 +1,14 @@
+# External taxonomy 2
+
+# comment
+en:coffee
+fr:café
+
+en:tomato
+fr:tomate
+
+<en:coffee
+en:cappuccino
+
+<en:coffee
+en:latte