refactor: avoid crashes when the new nutrition schema is deployed (#1837)

This is the first implementation step of #1836.

With this first milestone, the backend won't crash if Product Opener
uses the new schema version 1003.

Author: raphael0202

robotoff/insights/importer.py

@@ -1501,12 +1501,6 @@ def generate_candidates(
         predictions: list[Prediction],
         product_id: ProductIdentifier,
     ) -> Iterator[ProductInsight]:
-        if product and product.nutrition_data_prepared:
-            # Don't generate candidates if the product has nutrition
-            # information per prepared product, as the model doesn't
-            # handle this case
-            return
-
         image_orientation_prediction = next(
             (p for p in predictions if p.type == PredictionType.image_orientation),
             None,
@@ -1584,6 +1578,19 @@ def keep_prediction(product: Product | None, nutrients: JSONType) -> bool:
         :param nutrients: the nutrient values extracted from the image
         :return: True if the prediction should be kept, False otherwise
         """
+        if product and (
+            # Don't support nutrition extraction with the new `nutriments` schema
+            product.schema_version
+            > 1002
+        ):
+            return False
+
+        if product and product.nutrition_data_prepared:
+            # Don't generate candidates if the product has nutrition
+            # information per prepared product, as the model doesn't
+            # handle this case
+            return False
+
         if product is None or not product.nutriments:
             # We don't have access to MongoDB or the nutriment data is missing
             # completely, so we generate an insight
@@ -1720,6 +1727,10 @@ def add_optional_fields(cls, insight: ProductInsight, product: Product | None):
             # Stop here
             return
 
+        # Don't support nutrition extraction with the new `nutriments` schema
+        if product.schema_version > 1002:
+            return
+
         campaigns: list[str] = []
         if set(product.nutriments.keys()):
             # The product already has some nutrient values, so we add it to the

robotoff/metrics.py

@@ -1,6 +1,5 @@
 import datetime
 import logging
-from typing import Iterable, Iterator
 from urllib.parse import urlparse
 
 from influxdb_client import InfluxDBClient
@@ -276,49 +275,3 @@ def generate_insight_metrics(target_datetime: datetime.datetime) -> list[dict]:
             }
         )
     return inserts
-
-
-def generate_recent_changes_metrics(items: Iterable[dict]) -> Iterator[dict]:
-    for item in items:
-        comment: str = item["comment"]
-        diffs: dict = item["diffs"]
-        uploaded_images = diffs.setdefault("uploaded_images", {})
-        selected_images: dict = diffs.setdefault("selected_images", {})
-        nutriments: dict = diffs.setdefault("nutriments", {})
-        nutriments_add: dict = nutriments.get("add", {})
-        nutriments_change: dict = nutriments.get("change", {})
-        nutriments_delete: dict = nutriments.get("delete", {})
-        fields: dict = diffs.setdefault("fields", {})
-        fields_add: dict = fields.setdefault("add", {})
-        fields_change: dict = fields.setdefault("change", {})
-        packagings: dict = diffs.setdefault("packagings", {})
-        yield {
-            "measurement": "recent_changes",
-            "tags": {
-                "countries_tags": item["countries_tags"],
-                "user_id": item["userid"],
-                "is_smooth_app": int("Smoothie - OpenFoodFacts" in comment),
-                "by_robotoff": int("[robotoff]" in comment),
-                "has_image_upload": int(bool(uploaded_images.get("add", {}))),
-                "has_image_delete": int(bool(uploaded_images.get("delete", {}))),
-                "has_image_selection_change": int(
-                    bool(selected_images.get("change", {}))
-                ),
-                "has_image_selection_add": int(bool(selected_images.get("add", {}))),
-                "has_image_selection_delete": int(
-                    bool(selected_images.get("delete", {}))
-                ),
-                "has_nutriment_change": int(bool(nutriments_change)),
-                "has_nutriment_add": int(bool(nutriments_add)),
-                "has_nutriment_delete": int(bool(nutriments_delete)),
-                "has_nutriscore_added": int("nutrition-score-fr" in nutriments_add),
-                "has_nutriscore_change": int("nutrition-score-fr" in nutriments_change),
-                "has_nutriscore_delete": int("nutrition-score-fr" in nutriments_delete),
-                "has_categories_add": int("categories" in fields_add),
-                "has_categories_change": int("categories" in fields_change),
-                "has_packagings_add": int(bool(packagings.get("add", {}))),
-                "has_packagings_change": int(bool(packagings.get("change", {}))),
-            },
-            "time": item["t"],
-            "fields": {"count": 1},
-        }

robotoff/prediction/category/neural/keras_category_classifier_3_0/preprocessing.py

@@ -53,8 +53,8 @@ def generate_inputs_dict(
 ) -> JSONType:
     """Generate inputs for v3 category predictor model.
 
-    :param product: the product dict, the `product_name` and `ingredients`
-        fields are used, if provided
+    :param product: the product dict, the `product_name`, `ingredients`
+        and `nutriments` fields are used, if provided
     :param ocr_texts: a list of detected OCR texts, one per image
     :param image_embeddings: embeddings generated by CLIP model of up to
         the `MAX_IMAGE_EMBEDDING` most recent images.
@@ -75,12 +75,14 @@ def generate_inputs_dict(
         or [""],
     }
 
-    nutriments = product.get("nutriments", {})
-    for nutriment_name in NUTRIMENT_NAMES:
-        inputs[nutriment_name] = transform_nutrition_input(
-            nutriments.get(f"{nutriment_name.replace('_', '-')}_100g"),
-            nutriment_name=nutriment_name,
-        )
+    # Only support legacy nutrition schema for now
+    if product.get("schema_version", 999) <= 1002:
+        nutriments = product.get("nutriments", {})
+        for nutriment_name in NUTRIMENT_NAMES:
+            inputs[nutriment_name] = transform_nutrition_input(
+                nutriments.get(f"{nutriment_name.replace('_', '-')}_100g"),
+                nutriment_name=nutriment_name,
+            )
 
     if image_embeddings is None:
         # No image is available, so we provide zero-filled image embedding

robotoff/products.py

@@ -435,6 +435,7 @@ class Product:
         "nutrition_data_per",
         "nutrition_data_prepared",
         "serving_size",
+        "schema_version",
     )
 
     def __init__(self, product: JSONType):
@@ -474,6 +475,8 @@ def __init__(self, product: JSONType):
             product.get("nutrition_data_prepared") == "on"
         )
         self.serving_size: str | None = product.get("serving_size")
+        # if `schema_version` is not present, we assume it's 999
+        self.schema_version: int = product.get("schema_version", 999)
 
     @staticmethod
     def get_fields(item: JSONType) -> set[str]:

robotoff/scheduler/latent.py

@@ -49,12 +49,16 @@ def generate_fiber_quality_facet() -> None:
 
         product_id = ProductIdentifier(barcode, server_type)
         product = product_store.get_product(
-            product_id, ["nutriments", "data_quality_tags", "images"]
+            product_id, ["nutriments", "data_quality_tags", "images", "schema_version"]
         )
 
         if product is None:
             continue
 
+        # Don't support (yet) the new nutrition schema
+        if product.get("schema_version", 999) > 1002:
+            continue
+
         nutriments = product.get("nutriments", {})
         data_quality_tags = product.get("data_quality_tags", {})
         images = product.get("images", {})

robotoff/workers/tasks/product_updated.py

@@ -112,7 +112,9 @@ def should_rerun_category_predictor(diffs: JSONType | None) -> bool:
     fields = diffs.get("fields", {})
     updated_fields = fields.get("change", [])
     added_fields = fields.get("add", [])
-    has_nutriments_change = "nutriments" in diffs
+    # TODO(raphael): remove the `"nutriments" in diffs` once the new
+    # nutrition schema is fully rolled out.
+    has_nutriments_change = "nutriments" in diffs or "nutrition" in diffs
     uploaded_images = diffs.get("uploaded_images", {})
     is_uploaded_image = "add" in uploaded_images
     is_deleted_image = "delete" in uploaded_images