feat: support new folders structure (#434)

openfoodfacts/openfoodfacts-server@2b2d85f added a new structure to taxonomy folders.

This is a quick fix to support it

Author: alexgarel

backend/editor/api.py

@@ -374,7 +374,7 @@ async def import_from_github(
     taxonomy = TaxonomyGraph(branch, taxonomy_name)
 
     if not taxonomy.is_valid_branch_name():
-        raise HTTPException(status_code=422, detail="branch_name: Enter a valid branch name!")
+        raise HTTPException(status_code=422, detail="branch_name: Enter a valid branch name!")
     if await taxonomy.does_project_exist():
         raise HTTPException(status_code=409, detail="Project already exists!")
     if not await taxonomy.is_branch_unique(from_github=True):

backend/editor/entries.py

@@ -2,6 +2,7 @@
 Database helper functions for API
 """
 
+import logging
 import re
 import shutil
 import tempfile
@@ -15,7 +16,7 @@
 from openfoodfacts_taxonomy_parser import unparser  # Unparser for taxonomies
 from openfoodfacts_taxonomy_parser import utils as parser_utils
 
-from . import settings
+from . import settings, utils
 from .controllers.node_controller import create_entry_node
 from .controllers.project_controller import create_project, edit_project, get_project
 from .exceptions import GithubBranchExistsError  # Custom exceptions
@@ -33,7 +34,8 @@
 )
 from .models.node_models import EntryNodeCreate
 from .models.project_models import ProjectCreate, ProjectEdit, ProjectStatus
-from .utils import file_cleanup
+
+log = logging.getLogger(__name__)
 
 
 async def async_list(async_iterable):
@@ -48,6 +50,10 @@ def __init__(self, branch_name, taxonomy_name):
         self.branch_name = branch_name
         self.project_name = "p_" + taxonomy_name + "_" + branch_name
 
+    @property
+    def taxonomy_path_in_repository(self):
+        return utils.taxonomy_path_in_repository(self.taxonomy_name)
+
     def get_label(self, id):
         """
         Helper function for getting the label for a given id
@@ -82,13 +88,13 @@ async def get_local_taxonomy_file(self, tmpdir: str, uploadfile: UploadFile):
 
     async def get_github_taxonomy_file(self, tmpdir: str):
         async with TransactionCtx():
-            filename = f"{self.taxonomy_name}.txt"
-            filepath = f"{tmpdir}/{filename}"
-            base_url = (
-                "https://raw.githubusercontent.com/" + settings.repo_uri + "/main/taxonomies/"
+            filepath = f"{tmpdir}/{self.taxonomy_name}.txt"
+            target_url = (
+                f"https://raw.githubusercontent.com/{settings.repo_uri}"
+                f"/main/{self.taxonomy_path_in_repository}"
             )
             try:
-                await run_in_threadpool(urllib.request.urlretrieve, base_url + filename, filepath)
+                await run_in_threadpool(urllib.request.urlretrieve, target_url, filepath)
                 github_object = GithubOperations(self.taxonomy_name, self.branch_name)
                 commit_sha = (await github_object.get_branch("main")).commit.sha
                 file_sha = await github_object.get_file_sha()
@@ -131,6 +137,7 @@ async def get_and_parse_taxonomy(self, uploadfile: UploadFile | None = None):
             # add an error node so we can display it with errors in the app
             async with TransactionCtx():
                 await edit_project(self.project_name, ProjectEdit(status=ProjectStatus.FAILED))
+            log.exception(e)
             raise e
 
     async def import_taxonomy(
@@ -169,9 +176,10 @@ def dump_taxonomy(self, background_tasks: BackgroundTasks):
                 # Dump taxonomy with given file name and branch name
                 unparser_object(filename, self.branch_name, self.taxonomy_name)
                 # Program file removal in the background
-                background_tasks.add_task(file_cleanup, filename)
+                background_tasks.add_task(utils.file_cleanup, filename)
                 return filename
             except Exception as e:
+                log.exception(e)
                 raise TaxonomyUnparsingError() from e
 
     async def file_export(self, background_tasks: BackgroundTasks):

backend/editor/github_functions.py

@@ -16,7 +16,7 @@
     PullRequest,
 )
 
-from . import settings
+from . import settings, utils
 
 
 class GithubOperations:
@@ -26,6 +26,10 @@ def __init__(self, taxonomy_name: str, branch_name: str):
         self.taxonomy_name = taxonomy_name
         self.branch_name = branch_name
 
+    @property
+    def taxonomy_path_in_repository(self):
+        return utils.taxonomy_path_in_repository(self.taxonomy_name)
+
     @cached_property
     def repo_info(self) -> tuple[str, str]:
         repo_uri = settings.repo_uri
@@ -68,10 +72,9 @@ async def get_file_sha(self) -> str:
         """
         Get the file SHA from the 'main' branch in the "openfoodfacts-server" repo
         """
-        github_filepath = f"taxonomies/{self.taxonomy_name}.txt"
         file_contents: ContentFile = (
             await self.connection.rest.repos.async_get_content(
-                *self.repo_info, path=github_filepath
+                *self.repo_info, path=self.taxonomy_path_in_repository
             )
         ).parsed_data
 
@@ -90,8 +93,7 @@ async def update_file(self, filename: str, file_sha: str, author_name) -> FileCo
         Update the taxonomy txt file edited by user using the Taxonomy Editor
         """
         # Find taxonomy text file to be updated
-        github_filepath = f"taxonomies/{self.taxonomy_name}.txt"
-        commit_message = f"Update {self.taxonomy_name}.txt"
+        commit_message = f"Update {self.taxonomy_path_in_repository}"
         author = {"name": author_name, "email": "[email protected]"}
         try:
             with open(filename, "r") as f:
@@ -103,7 +105,7 @@ async def update_file(self, filename: str, file_sha: str, author_name) -> FileCo
             return (
                 await self.connection.rest.repos.async_create_or_update_file_contents(
                     *self.repo_info,
-                    path=github_filepath,
+                    path=self.taxonomy_path_in_repository,
                     message=commit_message,
                     content=base64.b64encode(new_file_contents.encode("utf-8")),
                     sha=file_sha,

backend/editor/utils.py

@@ -12,3 +12,12 @@ def file_cleanup(filepath):
         os.remove(filepath)
     except FileNotFoundError:
         log.warn(f"Taxonomy file {filepath} not found for deletion")
+
+
+def taxonomy_path_in_repository(taxonomy_name):
+    """Helper function to get the path of a taxonomy in the repository"""
+    path = taxonomy_name
+    # hacky for now until we restructure better
+    if path in ("food_ingredients", "food_categories"):
+        path = path.replace("_", "/")
+    return f"taxonomies/{path}.txt"

taxonomy-editor-frontend/src/constants.ts

@@ -27,11 +27,11 @@ export const TAXONOMY_NAMES = [
   "Additives",
   "Allergens",
   "Amino Acids",
-  "Categories",
+  "Food Categories",
   "Data Quality",
   "Food Groups",
   "Improvements",
-  "Ingredients",
+  "Food Ingredients",
   "Ingredients Analysis",
   "Ingredients Processing",
   "Labels",

taxonomy-editor-frontend/src/pages/startproject/index.tsx

@@ -32,7 +32,7 @@ export const StartProject = () => {
 
   const findDefaultBranchName = useCallback(() => {
     if (taxonomyName === "" || ownerName === "") return "";
-    return `${taxonomyName.toLowerCase()}_${ownerName
+    return `${toSnakeCase(taxonomyName.toLowerCase())}_${ownerName
       .replace(" ", "")
       .toLowerCase()}_${Math.floor(Date.now() / 1000)}`;
   }, [ownerName, taxonomyName]);