Merge pull request #4032 from Spinnich/fix/search-by-hash

fix(roms): search the gallery by CRC32, MD5, SHA-1 and RA hash

Author: gantoine

backend/alembic/versions/0106_hash_search_indexes.py

@@ -0,0 +1,43 @@
+"""Index the hash columns searched by the gallery search box
+
+Searching by a CRC32/MD5/SHA-1/RA digest now matches the hash columns on
+``roms`` and ``rom_files``. Neither table had an index on any of them, so every
+such search scanned both tables in full.
+
+Revision ID: 0106_hash_search_indexes
+Revises: 0105_fix_gamelist_epoch_ms
+Create Date: 2026-07-31 00:00:00.000000
+
+"""
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "0106_hash_search_indexes"
+down_revision = "0105_fix_gamelist_epoch_ms"
+branch_labels = None
+depends_on = None
+
+HASH_INDEXES: dict[str, tuple[str, ...]] = {
+    "roms": ("crc_hash", "md5_hash", "sha1_hash", "ra_hash"),
+    "rom_files": ("crc_hash", "md5_hash", "sha1_hash", "ra_hash", "chd_sha1_hash"),
+}
+
+
+def upgrade() -> None:
+    for table, columns in HASH_INDEXES.items():
+        with op.batch_alter_table(table, schema=None) as batch_op:
+            for column in columns:
+                batch_op.create_index(
+                    f"idx_{table}_{column}",
+                    [column],
+                    unique=False,
+                    if_not_exists=True,
+                )
+
+
+def downgrade() -> None:
+    for table, columns in HASH_INDEXES.items():
+        with op.batch_alter_table(table, schema=None) as batch_op:
+            for column in columns:
+                batch_op.drop_index(f"idx_{table}_{column}", if_exists=True)

backend/handler/database/roms_handler.py

@@ -25,6 +25,7 @@
     select,
     text,
     true,
+    union,
     update,
 )
 from sqlalchemy.orm import (
@@ -132,6 +133,27 @@
 # 3 is the default minimum size in InnoDB
 FULLTEXT_MIN_TOKEN_SIZE = 3
 
+# A term reaches the hash columns only when it is hex of exactly a digest
+# length, so an ordinary name search builds no hash SQL at all. Hashes are
+# stored lowercase, which keeps the lookup an indexed equality.
+HEX_DIGEST_REGEX = re.compile(r"[0-9a-fA-F]+")
+
+# CRC32 (8), MD5 and RetroAchievements (32), SHA-1 (40).
+ROM_HASH_COLUMNS_BY_DIGEST_LENGTH: dict[int, tuple[QueryableAttribute, ...]] = {
+    8: (Rom.crc_hash,),
+    32: (Rom.md5_hash, Rom.ra_hash),
+    40: (Rom.sha1_hash,),
+}
+
+# Multi-file games (multi-disc, multi-track) keep their hashes per file, which
+# is the hash a user has in hand. `chd_sha1_hash` is the uncompressed disc's
+# digest, the one datfiles publish for a CHD.
+ROM_FILE_HASH_COLUMNS_BY_DIGEST_LENGTH: dict[int, tuple[QueryableAttribute, ...]] = {
+    8: (RomFile.crc_hash,),
+    32: (RomFile.md5_hash, RomFile.ra_hash),
+    40: (RomFile.sha1_hash, RomFile.chd_sha1_hash),
+}
+
 # Filter dropdowns read the narrow `roms_facets` mirror instead of `roms`,
 # whose rows carry the raw metadata blobs. Column order matches the unpacking
 # in `_collect_filter_values`.
@@ -583,12 +605,8 @@ def _build_fulltext_relevance(self, search_term: str) -> str | None:
                 parts.append('"' + " ".join(words) + '"')
         return " ".join(parts) if parts else None
 
-    def _filter_by_search_term(self, query: Query, search_term: str):
-        terms = [term.strip() for term in search_term.split("|")]
-        terms = [term for term in terms if term]
-        if not terms:
-            return query
-
+    def _build_name_conditions(self, terms: Sequence[str]) -> list[Any]:
+        """Match the term against the ROM's name and filename."""
         if ROMM_DB_DRIVER in ("mariadb", "mysql"):
             match_clauses: list[Any] = []
             for idx, term in enumerate(terms):
@@ -604,7 +622,7 @@ def _filter_by_search_term(self, query: Query, search_term: str):
                     ).bindparams(**{param: boolean_query})
                 )
             if match_clauses:
-                return query.filter(or_(*match_clauses))
+                return match_clauses
 
         # psql and full-text fallback
         term_conditions = []
@@ -615,7 +633,56 @@ def _filter_by_search_term(self, query: Query, search_term: str):
             ]
             if word_conditions:
                 term_conditions.append(and_(*word_conditions))
-        return query.filter(or_(*term_conditions))
+        return term_conditions
+
+    def _build_hash_selects(self, terms: Iterable[str]) -> list[Select]:
+        """Id-yielding selects for terms shaped like a hash digest.
+
+        A ROM's own hashes and its files' are queried separately so each side
+        keeps its own index. Returns nothing when no term looks like a digest,
+        which is the case for every ordinary name search.
+        """
+        rom_predicates: list[ColumnElement[bool]] = []
+        file_predicates: list[ColumnElement[bool]] = []
+
+        for term in terms:
+            rom_columns = ROM_HASH_COLUMNS_BY_DIGEST_LENGTH.get(len(term))
+            if rom_columns is None or not HEX_DIGEST_REGEX.fullmatch(term):
+                continue
+            digest = term.lower()
+            rom_predicates.extend(column == digest for column in rom_columns)
+            file_predicates.extend(
+                column == digest
+                for column in ROM_FILE_HASH_COLUMNS_BY_DIGEST_LENGTH[len(term)]
+            )
+
+        if not rom_predicates:
+            return []
+
+        return [
+            select(Rom.id).where(or_(*rom_predicates)),
+            select(RomFile.rom_id.label("id")).where(or_(*file_predicates)),
+        ]
+
+    def _filter_by_search_term(self, query: Query, search_term: str):
+        terms = [term.strip() for term in search_term.split("|")]
+        terms = [term for term in terms if term]
+        if not terms:
+            return query
+
+        name_conditions = self._build_name_conditions(terms)
+        hash_selects = self._build_hash_selects(terms)
+        if not hash_selects:
+            return query.filter(or_(*name_conditions))
+
+        # OR-ing the hash columns onto the name conditions would cost the
+        # full-text index its only chance to drive the query, scanning `roms`
+        # end to end. Resolving each side through its own index and unioning
+        # the ids searches both without giving up either index.
+        matches = union(
+            select(Rom.id).where(or_(*name_conditions)), *hash_selects
+        ).subquery()
+        return query.filter(Rom.id.in_(select(matches.c.id)))
 
     def _filter_by_matched(self, query: Query, value: bool) -> Query:
         """Filter based on whether the rom is matched to a metadata provider.

backend/models/rom.py

@@ -106,7 +106,15 @@ class RomArchiveMember(TypedDict):
 class RomFile(BaseModel):
     __tablename__ = "rom_files"
 
-    __table_args__ = (Index("idx_rom_files_rom_id", "rom_id"),)
+    __table_args__ = (
+        Index("idx_rom_files_rom_id", "rom_id"),
+        # Searching the gallery by a hash digest
+        Index("idx_rom_files_crc_hash", "crc_hash"),
+        Index("idx_rom_files_md5_hash", "md5_hash"),
+        Index("idx_rom_files_sha1_hash", "sha1_hash"),
+        Index("idx_rom_files_ra_hash", "ra_hash"),
+        Index("idx_rom_files_chd_sha1_hash", "chd_sha1_hash"),
+    )
 
     id: Mapped[int] = mapped_column(primary_key=True, autoincrement=True)
     rom_id: Mapped[int] = mapped_column(ForeignKey("roms.id", ondelete="CASCADE"))
@@ -336,6 +344,11 @@ class Rom(BaseModel):
         Index("idx_roms_hltb_id", "hltb_id"),
         Index("idx_roms_gamelist_id", "gamelist_id"),
         Index("idx_roms_libretro_id", "libretro_id"),
+        # Searching the gallery by a hash digest
+        Index("idx_roms_crc_hash", "crc_hash"),
+        Index("idx_roms_md5_hash", "md5_hash"),
+        Index("idx_roms_sha1_hash", "sha1_hash"),
+        Index("idx_roms_ra_hash", "ra_hash"),
     )
 
     fs_name: Mapped[str] = mapped_column(String(length=FILE_NAME_MAX_LENGTH))