Merge pull request #3986 from rommapp/fix/launchbox-cloud-empty-store

fix(launchbox): stop the cloud provider from silently matching nothing

Author: gantoine

backend/endpoints/tasks.py

@@ -11,7 +11,6 @@
     ENABLE_RESCAN_ON_FILESYSTEM_CHANGE,
     RESCAN_ON_FILESYSTEM_CHANGE_DELAY,
     TASK_RESULT_TTL,
-    TASK_TIMEOUT,
 )
 from decorators.auth import protected_route
 from endpoints.responses import (
@@ -145,7 +144,7 @@ def _build_task_info(name: str, task: Task) -> TaskInfo:
         title=task.title,
         description=task.description,
         enabled=task.enabled,
-        manual_run=task.manual_run,
+        manual_run=task.can_run_manually,
         cron_string=task.cron_string or "",
     )
 
@@ -385,7 +384,7 @@ async def run_single_task(
         )
 
     task_instance = all_tasks[task_name]
-    if not task_instance.enabled or not task_instance.manual_run:
+    if not task_instance.can_run_manually:
         raise HTTPException(
             status_code=400,
             detail=f"Task '{task_name}' cannot be run",
@@ -394,7 +393,7 @@ async def run_single_task(
     job = low_prio_queue.enqueue(
         task_instance.run,
         kwargs=task_kwargs or {},
-        job_timeout=TASK_TIMEOUT,
+        job_timeout=task_instance.timeout,
         result_ttl=TASK_RESULT_TTL,
         meta={
             "task_name": task_instance.title,

backend/handler/metadata/launchbox_handler/handler.py

@@ -14,6 +14,7 @@
 from .remote_source import RemoteSource
 from .types import (
     DASH_COLON_REGEX,
+    LAUNCHBOX_METADATA_INITIAL_IMPORT_KEY,
     LAUNCHBOX_METADATA_NAME_KEY,
     LAUNCHBOX_PLATFORMS_DIR,
     LAUNCHBOX_TAG_REGEX,
@@ -39,8 +40,31 @@ def is_local_enabled(cls) -> bool:
     def is_enabled(cls) -> bool:
         return cls.is_cloud_enabled() or cls.is_local_enabled()
 
+    @staticmethod
+    async def is_remote_store_populated() -> bool:
+        return bool(await async_cache.exists(LAUNCHBOX_METADATA_NAME_KEY))
+
+    @staticmethod
+    async def is_remote_store_importing() -> bool:
+        return bool(await async_cache.exists(LAUNCHBOX_METADATA_INITIAL_IMPORT_KEY))
+
     async def heartbeat(self) -> bool:
-        return self.is_enabled()
+        if self.is_local_enabled():
+            return True
+
+        if not self.is_cloud_enabled():
+            return False
+
+        # A first import commits in batches, so the store starts answering for a
+        # handful of names long before it holds the whole dump. Until that run
+        # finishes, most lookups still miss.
+        if await self.is_remote_store_importing():
+            return False
+
+        # Cloud lookups read from a cache the metadata update task fills. Until
+        # it has run, every lookup returns nothing, so reporting healthy here
+        # would be a lie.
+        return await self.is_remote_store_populated()
 
     def get_platform(self, slug: str) -> LaunchboxPlatform:
         return get_platform(slug)
@@ -60,9 +84,7 @@ async def get_rom(
 
         local = await self._local.get_rom(fs_name, platform_slug)
 
-        remote_available = remote_enabled and bool(
-            await async_cache.exists(LAUNCHBOX_METADATA_NAME_KEY)
-        )
+        remote_available = remote_enabled and await self.is_remote_store_populated()
 
         if local is not None:
             launchbox_id_local = safe_int(local.get("DatabaseID"))
@@ -238,6 +260,13 @@ async def get_matched_roms_by_name(
         if not self.is_enabled():
             return []
 
+        if self.is_cloud_enabled() and not await self.is_remote_store_populated():
+            log.warning(
+                "LaunchBox metadata store is empty, so no cloud results can be "
+                "returned. Set ENABLE_SCHEDULED_UPDATE_LAUNCHBOX_METADATA=true and "
+                "run the LaunchBox metadata update task to populate it."
+            )
+
         rom = await self.get_rom(search_term, platform_slug, keep_tags=True)
         return [rom]
 

backend/handler/metadata/launchbox_handler/platforms.py

@@ -52,7 +52,7 @@ class SlugToLaunchboxId(TypedDict):
     UPS.CAMPUTERS_LYNX: {"id": 61, "name": "Camputers Lynx"},
     UPS.CASIO_LOOPY: {"id": 114, "name": "Casio Loopy"},
     UPS.CASIO_PV_1000: {"id": 115, "name": "Casio PV-1000"},
-    UPS.COLECOADAM: {"id": 117, "name": "Coleco Adam"},
+    UPS.COLECOADAM: {"id": 117, "name": "Coleco ADAM"},
     UPS.COLECOVISION: {"id": 13, "name": "ColecoVision"},
     UPS.COLOUR_GENIE: {"id": 73, "name": "EACA EG2000 Colour Genie"},
     UPS.COMMODORE_CDTV: {"id": 120, "name": "Commodore CDTV"},
@@ -76,7 +76,7 @@ class SlugToLaunchboxId(TypedDict):
         "id": 58,
         "name": "Fairchild Channel F",
     },
-    UPS.FAMICOM: {"id": 27, "name": "Nintendo Famicom"},
+    UPS.FAMICOM: {"id": 27, "name": "Nintendo Entertainment System"},
     UPS.FDS: {"id": 157, "name": "Nintendo Famicom Disk System"},
     UPS.FM_7: {"id": 186, "name": "Fujitsu FM-7"},
     UPS.FM_TOWNS: {"id": 124, "name": "Fujitsu FM Towns Marty"},
@@ -161,7 +161,7 @@ class SlugToLaunchboxId(TypedDict):
     UPS.SEGACD: {"id": 39, "name": "Sega CD"},
     UPS.SEGACD32: {"id": 173, "name": "Sega CD 32X"},
     UPS.SERIES_X_S: {"id": 222, "name": "Microsoft Xbox Series X/S"},
-    UPS.SFAM: {"id": 53, "name": "Super Famicom"},
+    UPS.SFAM: {"id": 53, "name": "Super Nintendo Entertainment System"},
     UPS.SG1000: {"id": 80, "name": "Sega SG-1000"},
     UPS.SHARP_MZ_80B20002500: {"id": 205, "name": "Sharp MZ-2500"},
     UPS.SHARP_X68000: {"id": 128, "name": "Sharp X68000"},

backend/handler/metadata/launchbox_handler/remote_source.py

@@ -70,8 +70,14 @@ async def get_rom(
             metadata_database_index_entry = await async_cache.hget(
                 LAUNCHBOX_METADATA_DATABASE_ID_KEY, database_id
             )
-            if metadata_database_index_entry:
-                return json.loads(metadata_database_index_entry)
+            if not metadata_database_index_entry:
+                continue
+
+            # The alternate name index is not keyed by platform, so a hit can
+            # point at a same-titled game on a completely different system.
+            entry = json.loads(metadata_database_index_entry)
+            if entry.get("Platform") == platform_name:
+                return entry
 
         return None
 

backend/handler/metadata/launchbox_handler/types.py

@@ -16,6 +16,10 @@
 LAUNCHBOX_METADATA_IMAGE_KEY: Final[str] = "romm:launchbox_metadata_image"
 LAUNCHBOX_MAME_KEY: Final[str] = "romm:launchbox_mame"
 LAUNCHBOX_FILES_KEY: Final[str] = "romm:launchbox_files"
+# Set while the store is being filled for the first time
+LAUNCHBOX_METADATA_INITIAL_IMPORT_KEY: Final[str] = (
+    "romm:launchbox_metadata_initial_import"
+)
 
 LAUNCHBOX_LOCAL_DIR: Final[Path] = Path(ROMM_BASE_PATH) / "launchbox"
 LAUNCHBOX_PLATFORMS_DIR: Final[Path] = LAUNCHBOX_LOCAL_DIR / "Data" / "Platforms"