diff --git a/README.md b/README.md index 617fccd86..c088f1056 100644 --- a/README.md +++ b/README.md @@ -392,6 +392,34 @@ python -m client.query_runner - **GET /search** - Card search with query parameter support - **GET /favicon.ico** - Favicon for web interface +### Scryfall-Compatible Endpoints + +Every route Scryfall documents under `/cards`, answering with Scryfall's own response objects and +175-per-page pagination, so a client can be pointed here by swapping its base URL: + +- **GET /cards** and **GET /cards/search** - list routes (`format=json|csv`) +- **GET /cards/named**, **/cards/autocomplete**, **/cards/random** - single-card and catalog lookups +- **POST /cards/collection** - up to 75 identifiers at once +- **GET /cards/:id**, **/cards/:code/:number(/:lang)**, and the multiverse / mtgo / arena / + tcgplayer / cardmarket id namespaces +- **GET /cards/:id/rulings** and its four sibling addressings + +`format=text` and `format=image` are available on the single-card routes. What is *not* identical — +chiefly that the corpus is a filtered subset of Scryfall's — is recorded in +[docs/issues/local-scryfall-cards-api.md](docs/issues/local-scryfall-cards-api.md). + +The reference half of the API, mirrored from Scryfall rather than derived from the corpus: + +- **GET /sets**, **/sets/:code**, **/sets/:id**, **/sets/tcgplayer/:id** - 1,047 Set objects +- **GET /catalog/:name** - all twenty catalogs, 62,187 values +- **GET /symbology** - 84 card symbols +- **GET /symbology/parse-mana?cost=** - computed, not stored, so it answers before the first import + +These are mirrored because the corpus cannot answer them: a Set object carries eight fields no card +carries, `card_count` counts printings this instance never imported, and a symbol's `svg_uri` exists +nowhere in the card data. Details in +[docs/issues/local-scryfall-sets-catalogs-symbology.md](docs/issues/local-scryfall-sets-catalogs-symbology.md). + ### Admin Endpoints Data-management routes — importing card data, running score/tag backfills, applying schema diff --git a/api/admin_resource.py b/api/admin_resource.py index 45ce865af..7a320e663 100644 --- a/api/admin_resource.py +++ b/api/admin_resource.py @@ -46,7 +46,11 @@ from api.card_processing import preprocess_card from api.db.bulk_upsert import bulk_upsert as _bulk_upsert +from api.rulings_import import import_rulings as _import_rulings from api.scryfall_bulk_data_fetcher import BulkDataKey, ScryfallBulkDataFetcher +from api.scryfall_reference_import import import_catalogs as _import_catalogs +from api.scryfall_reference_import import import_sets as _import_sets +from api.scryfall_reference_import import import_symbology as _import_symbology from api.settings import settings from api.tag_import import import_art_tags as _import_art_tags from api.tag_import import import_oracle_tags as _import_oracle_tags @@ -399,6 +403,13 @@ def _run_import_under_lock(self) -> None: self.backfill_prefer_scores() self.backfill_cubecobra_scores() _import_oracle_tags(self.app_context.writer_pool, self._bulk_data_fetcher) + # Rulings feed only /cards/*/rulings, so nothing above or below depends on them; they + # sit here rather than in their own pass so one bulk fetch cycle refreshes everything. + self._import_rulings_quietly() + # Alongside the rulings and for the same reason: reference data nothing else in this + # sequence reads, refreshed on the same cycle so one pass brings the whole surface up + # to date rather than leaving /sets and /catalog to age until a manual call. + self._import_reference_quietly() self.app_context.reload_engine(force=True) self._clear_caches() self.app_context.last_import_time.value = time.time() @@ -931,6 +942,72 @@ def import_art_tags(self, **_: object) -> dict[str, Any]: """Import art tags from Scryfall bulk data into art_tags, art_tag_relationships, and card_art_tags.""" return _import_art_tags(self.app_context.writer_pool, self._bulk_data_fetcher) + @route() + def import_rulings(self, **_: object) -> dict[str, Any]: + """Import Scryfall rulings bulk data into magic.rulings, backing the /cards/*/rulings routes. + + Returns: + The number of rulings loaded. + """ + return {"rulings_loaded": _import_rulings(self.app_context.writer_pool, self._bulk_data_fetcher)} + + def _import_rulings_quietly(self) -> None: + """Refresh the rulings during a bulk import, logging rather than failing on error. + + Rulings are the only data in the import sequence nothing else reads: a card search, the + prefer scores and the engine reload all work without them. Letting a bad rulings file + abort the import would cost the corpus refresh to save a rulings refresh. + """ + try: + _import_rulings(self.app_context.writer_pool, self._bulk_data_fetcher) + except Exception: + logger.exception("Rulings import failed; continuing with the rest of the import") + + @route() + def import_sets(self, **_: object) -> dict[str, Any]: + """Mirror Scryfall's set list into magic.sets, backing the /sets routes. + + Returns: + A summary of the load. + """ + return _import_sets(self.app_context.writer_pool, self._bulk_data_fetcher) + + @route() + def import_catalogs(self, **_: object) -> dict[str, Any]: + """Mirror the twenty Scryfall catalogs into magic.catalogs, backing /catalog/*. + + Returns: + A summary of the load. + """ + return _import_catalogs(self.app_context.writer_pool, self._bulk_data_fetcher) + + @route() + def import_symbology(self, **_: object) -> dict[str, Any]: + """Mirror Scryfall's card symbols into magic.card_symbols, backing /symbology. + + Returns: + A summary of the load. + """ + return _import_symbology(self.app_context.writer_pool, self._bulk_data_fetcher) + + def _import_reference_quietly(self) -> None: + """Refresh sets, catalogs and symbology during a bulk import, logging rather than failing. + + Each of the three is independent of the other two and of everything else in the sequence, + so one failing upstream endpoint must not cost the other two their refresh — nor the corpus + its. This is the rulings argument applied three more times: nothing downstream reads any of + these tables, so a stale one degrades three endpoints rather than breaking the import. + """ + for name, step in ( + ("Set", _import_sets), + ("Catalog", _import_catalogs), + ("Symbology", _import_symbology), + ): + try: + step(self.app_context.writer_pool, self._bulk_data_fetcher) + except Exception: + logger.exception("%s import failed; continuing with the rest of the import", name) + @route() def import_all_is_tags(self, **_: object) -> dict[str, Any]: """Discover and import all is: tags from Scryfall syntax documentation. diff --git a/api/api_resource.py b/api/api_resource.py index aa2aa637b..1a449f214 100644 --- a/api/api_resource.py +++ b/api/api_resource.py @@ -18,13 +18,14 @@ from typing import TYPE_CHECKING, Any, NoReturn import falcon +import falcon.util import orjson import psycopg from cachebox import LRUCache, TTLCache from api.admin_resource import ADMIN_MOUNT_PREFIX, AdminContext, AdminResource from api.app_context import AppContext -from api.enums import CardOrdering, PreferOrder, ResponseShape, SortDirection, UniqueOn +from api.enums import CardOrdering, PreferOrder, ResponseShape, SortDirection, UniqueOn, resolve_direction from api.middlewares.timing import record_span from api.noscript_helpers import generate_results_count_html, generate_results_html from api.parsing import generate_sql_query, parse_scryfall_query @@ -34,6 +35,7 @@ QueryBudgetExceeded, bounded_query_log_context, ) +from api.scryfall_compat import ScryfallCardsRoutes, ScryfallReferenceRoutes from api.settings import settings from api.utils import db_utils, error_monitoring from api.utils.css_utils import build_critical_css @@ -95,7 +97,39 @@ def _raise_query_bad_request(*, exc_name: str, query: str, description: str, err # Query parameters that must not be forwarded to action handlers. -DISALLOWED_QUERY_ARGS: frozenset[str] = frozenset(["falcon_response", "request_host"]) +# The route keys that make up the SCRYFALL-COMPATIBLE surface. +# +# It decides one thing: which shape a DISPATCH-level error takes on that path -- Scryfall's +# `{object, code, status, details}` or falcon's `{title, description}`. Everything a handler answers +# for itself already knows which surface it is on. +# +# The split is by ROUTE KEY rather than by path prefix because the two surfaces interleave under one +# namespace: `catalog` is Scryfall's `/catalog/:name` and `get_catalog` is this service's own, and +# only the table can tell them apart. Keys for routes a given branch has not merged yet are harmless +# -- an unregistered key is never resolved, so it is never consulted. +# +# What is deliberately NOT here: `_root`, `card`, `index`, `search` and `random_search` -- this +# service's own surface, whose error bodies its web interface renders by reading `title` and +# `description` -- plus `get_catalog`, `get_pid` and the admin handlers. +SCRYFALL_SURFACE_ROUTES = frozenset( + { + "cards", + "cards/search", + "cards/named", + "cards/autocomplete", + "cards/random", + "cards/collection", + "catalog", + "sets", + "symbology", + "symbology/parse-mana", + }, +) + +# Scryfall's sentence for a path that addresses nothing, measured 2026-08-16. +_SCRYFALL_NOT_FOUND_DETAILS = "The requested object or REST method was not found." + +DISALLOWED_QUERY_ARGS: frozenset[str] = frozenset(["falcon_response", "request", "request_host"]) # Body for an unhandled exception. Fixed and content-free on purpose: the frames live at throw sites # inside query and import paths, so their locals can hold connection and query state. Diagnostics go @@ -122,6 +156,25 @@ def pagination_ceiling() -> int: return int((time.time() - PAGINATION_BASE_TIMESTAMP) // PAGINATION_GROWTH_INTERVAL_SECONDS) +# `order=color`, as SQL. The eleven buckets Scryfall sorts colour into, measured 2026-08-09 over 923 +# cards spanning every colour shape: mono WUBRG, then multicolour by HOW MANY colours (guild pairs +# tie), then colourless, then lands. Two of those are not what a colour bitmask would give -- the +# colourless bucket sorts last rather than first, and lands after it -- which is why this is a CASE +# rather than an expression over card_colors. Mirrors color_sort_rank in card_engine/src/lib.rs; the +# two must agree or the SQL and engine paths order the same query differently. +_COLOR_ORDER_SQL = """ + (CASE + WHEN card_colors = '{"W": true}'::jsonb THEN 0 + WHEN card_colors = '{"U": true}'::jsonb THEN 1 + WHEN card_colors = '{"B": true}'::jsonb THEN 2 + WHEN card_colors = '{"R": true}'::jsonb THEN 3 + WHEN card_colors = '{"G": true}'::jsonb THEN 4 + WHEN (SELECT count(1) FROM jsonb_object_keys(card_colors)) > 1 + THEN 3 + (SELECT count(1) FROM jsonb_object_keys(card_colors)) + WHEN card_types ? 'Land' THEN 10 + ELSE 9 + END)""" + RESULT_FIELD_COLUMNS: dict[str, str] = { "name": "card_name", "set_code": "card_set_code", @@ -135,6 +188,11 @@ def pagination_ceiling() -> int: "illustration_id": "illustration_id", "scryfall_id": "scryfall_id", "price_usd": "price_usd", + # The other two currencies CardOrdering already sorts by (see the sql_orderby map's + # EUR/TIX entries). Without these a caller can rank a page by EUR or TIX and then have + # no way to read the number it was ranked on. Both are real magic.cards columns. + "price_eur": "price_eur", + "price_tix": "price_tix", "prefer_score": "prefer_score", # Card-data fields consumers need to run their own downstream filtering # (Scryfall JSON names and shapes): layout and rarity are plain text, @@ -213,6 +271,29 @@ def rewrap(query: str) -> str: return " ".join(query.strip().split()) +def _request_injection(entry: BoundRoute | None, req: falcon.Request) -> dict[str, Any]: + """Return the `request` keyword for handlers that declare it, and nothing for the rest. + + Only `POST /cards/collection` wants the request object: its identifiers arrive in the body, + which nothing else in the dispatch path reads. Injecting it unconditionally is not an option — + a non-string keyword a handler neither declares nor absorbs through `**kwargs` reaches it as a + TypeError, and `search` is one such handler. + + Args: + entry: The resolved route, or None when the path identified nothing. + req: The request being dispatched. + + Returns: + `{"request": req}` when the handler declares the parameter, otherwise an empty dict. + """ + if entry is None: + return {} + binder = getattr(entry.action, "binder", None) + if binder is None or not binder.accepts("request"): + return {} + return {"request": req} + + def _columnarize_cards(cards: list[dict[str, Any]]) -> dict[str, list[Any]]: """Convert a list of card dicts into a dict of per-field value lists. @@ -247,8 +328,16 @@ def _copy_query_result(result: dict[str, Any]) -> dict[str, Any]: return copied -class APIResource: - """Class implementing request handling for our simple API.""" +class APIResource(ScryfallCardsRoutes, ScryfallReferenceRoutes): + """Class implementing request handling for our simple API. + + The Scryfall-compatible routes live in the base classes rather than here: they are a + self-contained compatibility surface with their own response objects, and `iter_marked_routes` + scans inherited attributes, so they register exactly like the routes defined below. They are + two mixins rather than one because they answer from different places — `ScryfallCardsRoutes` + from the corpus and the engine, `ScryfallReferenceRoutes` from the tables mirrored off + api.scryfall.com — and share only the response plumbing in `ScryfallResponder`. + """ def __init__( self, @@ -336,6 +425,10 @@ def _build_action_kwargs(self, req: falcon.Request, resp: falcon.Response, entry Keyword arguments for the action call. """ params = {k: v for k, v in req.params.items() if k not in DISALLOWED_QUERY_ARGS} + # The request object itself, for the one handler that declares it (POST /cards/collection + # reads its identifiers from the body). Only where declared: a non-string keyword reaches a + # handler that neither declares nor absorbs it as a TypeError. + params.update(_request_injection(entry, req)) if entry is None: # Only _raise_not_found reads this; set after the query string so a request can't # spoof it via ?admin_authenticated=1 on a path that doesn't resolve to anything. @@ -344,7 +437,7 @@ def _build_action_kwargs(self, req: falcon.Request, resp: falcon.Response, entry params["request_host"] = req.get_header("X-Proxy-Host") or req.host return params - def _handle(self, req: falcon.Request, resp: falcon.Response) -> None: + def _handle(self, req: falcon.Request, resp: falcon.Response) -> None: # noqa: PLR0912 """Handle a Falcon request and set the response. Args: @@ -368,11 +461,31 @@ def _handle(self, req: falcon.Request, resp: falcon.Response) -> None: entry, action_args = self._resolve_action(path) action = self._raise_not_found - if entry is not None: + if entry is None and not req.context.get("admin_authenticated", False): + # AN UNKNOWN PATH ANSWERS IN SCRYFALL'S SHAPE, not with the route listing. + # + # The listing is a convenience for a human poking at the origin. A client is not a human: + # it parses `code` and `details`, and `{"title": ..., "description": {"routes": ...}}` + # gives it neither -- so a client pointed at this service instead of api.scryfall.com has + # to special-case this origin, which is exactly what it cannot do and still be pointable + # back at Scryfall. Status, wording and tier are measured (404, "The requested object or + # REST method was not found.", `no-cache`). + # + # A human still has the listing: every route is documented, and the route table is the + # source both this and `build_routes_listing` read -- and a caller who has proven they + # hold the admin secret (#966) still gets the full listing below, exactly as upstream + # answers them: the Scryfall shape is for the client that cannot present one. + self._respond_scryfall_error(resp, code="not_found", status=404, details=_SCRYFALL_NOT_FOUND_DETAILS) + return + if entry is None: + # Admin-authenticated unknown path: upstream's listing, the full one. + pass + elif req.method not in entry.spec.methods: # A route answers only the methods it declares. Checked after the path resolves, so a # path that identifies nothing stays a 404 rather than reporting what it would accept. - if req.method not in entry.spec.methods: - raise falcon.HTTPMethodNotAllowed(allowed_methods=sorted(entry.spec.methods)) + self._reject_method(resp, path=path, allowed=sorted(entry.spec.methods)) + return + else: action = entry.action res = None @@ -444,6 +557,59 @@ def _raise_not_found(self, *_args: object, admin_authenticated: bool = False, ** }, ) + def _reject_method(self, resp: falcon.Response, *, path: str, allowed: list[str]) -> None: + """Answer a method this route does not accept. + + On the Scryfall surface this is a 404 carrying the ordinary `not_found` object, with NO + `Allow` header -- which is what api.scryfall.com answers, measured 2026-08-16 across eight + requests: POST, PUT, DELETE and PATCH against `/cards/search`, `/cards/named`, + `/cards/collection`, `/cards/:id` and `/sets`. Not one of them carries `Allow`. + + 405 is the more correct HTTP answer in the abstract, and it is deliberately not used here. + Sending one would have meant inventing an error `code` for it, since Scryfall never emits a + 405 and there is therefore nothing to measure -- and an error body nobody checked is the same + defect as a column set nobody checked. A client that branches on 404-versus-405 has to see + what Scryfall shows it. + + This service's OWN routes keep falcon's 405 and its `Allow`: nothing there is mirroring + Scryfall, and 405 remains right for a route that genuinely declares its methods. + + Args: + resp: The response to write to. + path: The resolved route key, which selects the answer. + allowed: The methods this route does accept, sorted. + + Raises: + falcon.HTTPMethodNotAllowed: On this service's own surface, whose behaviour is unchanged. + """ + if path not in SCRYFALL_SURFACE_ROUTES: + raise falcon.HTTPMethodNotAllowed(allowed_methods=allowed) + self._respond_scryfall_error(resp, code="not_found", status=404, details=_SCRYFALL_NOT_FOUND_DETAILS) + + @staticmethod + def _respond_scryfall_error(resp: falcon.Response, *, code: str, status: int, details: str) -> None: + """Write a dispatch-level error in Scryfall's shape. + + Written directly rather than raised: falcon's error serializer produces `{title, + description}` from an `HTTPError`, which is the shape this is replacing. Indented, like every + `object: "error"` body api.scryfall.com sends, and `no-cache`, which is the tier it sends on + a 404 about a PATH (a 404 about DATA, such as `/sets/zzzz`, keeps the data tier and is + answered by its own route rather than here). + + Args: + resp: The response to write to. + code: Scryfall's error code. + status: The HTTP status, which the body repeats. + details: The human-readable sentence. + """ + resp.status = falcon.util.code_to_http_status(status) + resp.content_type = "application/json; charset=utf-8" + resp.set_header("Cache-Control", "no-cache") + resp.text = orjson.dumps( + {"object": "error", "code": code, "status": status, "details": details}, + option=orjson.OPT_INDENT_2, + ).decode() + def _run_query( self, *, @@ -815,6 +981,13 @@ def _search_engine( # noqa: PLR0913 offset: int = DEFAULT_OFFSET, fields: Sequence[str] | None = None, ) -> dict[str, Any]: + # AUTO is a request-level spelling neither search path knows, resolved on the way in so + # nothing downstream can see it. Resolved in each path rather than once in `_search` + # because what AUTO means depends on `orderby`: doing it here is necessarily after + # everything upstream that can still change `orderby` -- today nothing, once the in-query + # directives land their fold. Resolving before that fold would answer `order:usd` with the + # default ordering's direction and hand the engine the literal "auto". + direction = resolve_direction(direction, orderby) logger.info("Searching engine for %r", query) query_explanation = parsed_query.to_human_explanation() if query else "" try: @@ -862,6 +1035,13 @@ def _search_sql( # noqa: PLR0913 offset: int = DEFAULT_OFFSET, fields: Sequence[str] | None = None, ) -> dict[str, Any]: + # AUTO is a request-level spelling neither search path knows, resolved on the way in so + # nothing downstream can see it. Resolved in each path rather than once in `_search` + # because what AUTO means depends on `orderby`: doing it here is necessarily after + # everything upstream that can still change `orderby` -- today nothing, once the in-query + # directives land their fold. Resolving before that fold would answer `order:usd` with the + # default ordering's direction and hand the engine the literal "auto". + direction = resolve_direction(direction, orderby) logger.info("Searching SQL for %r", query) resolved_fields = self._resolve_result_fields(fields) query_explanation = parsed_query.to_human_explanation() if query else "" @@ -880,7 +1060,18 @@ def _search_sql( # noqa: PLR0913 CardOrdering.RARITY: "card_rarity_int", CardOrdering.TOUGHNESS: "creature_toughness", CardOrdering.USD: "price_usd", + CardOrdering.EUR: "price_eur", + CardOrdering.TIX: "price_tix", CardOrdering.CUBECOBRA: "cubecobra_score", + CardOrdering.RELEASED: "released_at", + # lower() for the same reason as name: the engine ranks the lowercased artist, and set + # codes are stored lowercase but nothing constrains them to be. + CardOrdering.ARTIST: "lower(card_artist)", + CardOrdering.SET: "lower(card_set_code)", + # Scryfall's colour order is eleven buckets, not the colour bitmask -- WUBRG, then + # multicolour by how many colours, then colourless, then lands. Measured 2026-08-09; + # mirrors color_sort_rank in card_engine/src/lib.rs, which the engine path uses. + CardOrdering.COLOR: _COLOR_ORDER_SQL, }.get(orderby, "edhrec_rank") sql_direction = { "asc": "ASC", diff --git a/api/card_processing.py b/api/card_processing.py index 3de650c86..ad5dc0cff 100644 --- a/api/card_processing.py +++ b/api/card_processing.py @@ -85,6 +85,142 @@ def extract_collector_number_int(collector_number: str | int | float | None) -> return None # Field will be null by default +# Face-merge policy for multi-face cards (#400, #873). Scryfall AND's search predicates at the +# CARD level, each satisfiable by any face — measured against api.scryfall.com 2026-08-08: +# `t:sorcery t:land` returns the MDFC lands (no single face is both), o: conjunctions match +# across faces (Ral, Monsoon Mage), and `c:b` matches Westvale Abbey's back-face-only color. +# One row per printing carrying any-face unions reproduces those semantics directly; one row +# per face would instead break every cross-face conjunction (no face-row satisfies both terms) +# on top of colliding on the scryfall_id primary key, which is how the back face silently won +# until now. Front-face scalars (cmc, mana cost, illustration, image, prices) match Scryfall's +# own top-level fields, verified on its card objects. +_FACE_LIST_UNIONS = ("card_types", "card_subtypes") +_FACE_FLAG_UNIONS = ("card_colors", "card_keywords", "produced_mana") +_FACE_JOINED_TEXTS = ("oracle_text", "flavor_text", "type_line") +# Copied per GROUP from the first face that has any of the group, so the numeric columns and +# their _text twins always describe the same face (the schema's check constraints couple them). +_FACE_STAT_GROUPS = ( + ("creature_power", "creature_toughness", "creature_power_text", "creature_toughness_text"), + ("planeswalker_loyalty", "planeswalker_loyalty_text"), +) +# Joins face texts. "\n" so substring/regex matches cannot span faces in practice (`.` does not +# cross newlines), "//" because that is the face separator Scryfall itself renders. +_FACE_TEXT_SEPARATOR = "\n//\n" + +# What `card_faces` stores per face, in Scryfall's own key names and value shapes. +# +# The merged row above is what the query planner filters on; this is what a face IS. Keeping it +# structurally (rather than inside raw_card_blob) is what lets the ENGINE answer face-level +# questions: the store is the only thing an engine-path request reads, and a JSONB column is not +# in it. It also retires the merge's one documented residual — when several faces carry a stat +# group (Brutal Cathar's 2/2 // 3/3) the merged row keeps only the front's, while Scryfall matches +# either; per-face power/toughness/loyalty make the back searchable again. +# +# `object` is the constant "card_face" and `image_uris` is a pure function of the card's id and the +# face's position, so neither is stored; both are re-emitted on read. +_FACE_OBJECT_FIELDS = ( + "name", + "mana_cost", + "type_line", + "oracle_text", + "power", + "toughness", + "loyalty", + # Battles print their defense on the FACE (Invasion of Alara's front face is `defense: 7`) and + # no column holds it, so leaving it out drops the number from every battle's card object. + "defense", + "colors", + "color_indicator", + "flavor_text", + "artist", + "artist_id", + "illustration_id", +) + + +def _face_records(card_faces: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Snapshot each face's own fields, front first. + + Args: + card_faces: The card's raw `card_faces` array, as Scryfall sent it. + + Returns: + One dict per face, carrying only the keys the face actually has. Absent keys stay absent + rather than becoming null, because Scryfall omits them and a reconstructed face has to + agree key-for-key. + """ + return [{field: face[field] for field in _FACE_OBJECT_FIELDS if field in face} for face in card_faces] + + +# Keys that do NOT go in card_compat_blob, because a column already holds them or they are a pure +# function of one. Kept subtractive, and mirrored in 2026-08-10-01-engine-card-objects.sql: the +# residue is "whatever is left", so a Scryfall key nobody has seen yet lands in the blob by default +# instead of being silently dropped the first time it appears. +# +# `prices` is deliberately absent from this set even though price_usd/eur/tix are columns -- +# usd_foil, usd_etched and eur_foil are not, and keeping the object whole costs a few bytes against +# losing three fields. +_COMPAT_BLOB_EXCLUDED = frozenset( + { + # stored in a column of their own + "id", "oracle_id", "name", "released_at", "layout", "mana_cost", "cmc", "type_line", + "oracle_text", "power", "toughness", "loyalty", "colors", "color_identity", "keywords", + "set", "set_name", "collector_number", "rarity", "flavor_text", "artist", + "illustration_id", "border_color", "edhrec_rank", "legalities", "produced_mana", + "watermark", "reserved", "game_changer", "frame", + # pure functions of id / set / collector_number / oracle_id, re-emitted on read + "object", "uri", "scryfall_uri", "image_uris", "rulings_uri", "prints_search_uri", + "set_uri", "set_search_uri", "scryfall_set_uri", "card_back_id", "related_uris", + "purchase_uris", "resource_id", + # its own column + "card_faces", + # added by this module before the snapshot is taken + "card_name", "face_name", "face_idx", "scryfall_id", + }, +) # fmt: skip + + +def _compat_blob(card: dict[str, Any]) -> dict[str, Any]: + """The Scryfall keys that no column holds and no derivation recovers. + + Args: + card: The card object as Scryfall sent it, before this module's own keys matter. + + Returns: + The residue, ready to store as card_compat_blob. + """ + return {key: value for key, value in card.items() if key not in _COMPAT_BLOB_EXCLUDED} + + +def _merge_processed_faces(faces: list[dict[str, Any]]) -> dict[str, Any]: + """Collapse fully-processed per-face rows into the card's single searchable row. + + The first face (the front) supplies the row and with it every identity and display + scalar; later faces fold in per the policy tables above. Known residual, sized in + the tests: when several faces carry a stat group (Brutal Cathar's 2/2 // 3/3), only + the first face's values are searchable — Scryfall also matches the back's. + + Args: + faces: Non-empty list of processed rows, one per surviving face, front first. + + Returns: + The merged row (the front face's dict, mutated in place). + """ + merged, *rest = faces + for face in rest: + for key in _FACE_LIST_UNIONS: + seen = merged[key] + seen.extend(value for value in face[key] if value not in seen) + for key in _FACE_FLAG_UNIONS: + merged[key].update(face[key]) + for key in _FACE_JOINED_TEXTS: + parts = [part for part in (merged.get(key), face.get(key)) if part] + merged[key] = (" // " if key == "type_line" else _FACE_TEXT_SEPARATOR).join(parts) + for group in _FACE_STAT_GROUPS: + if all(merged.get(field) is None for field in group) and any(face.get(field) is not None for field in group): + for field in group: + merged[field] = face.get(field) + return merged def extract_frame_data_from_raw_card(raw_card: dict) -> dict[str, bool]: """Extract frame data from a raw card dictionary. @@ -115,8 +251,9 @@ def extract_frame_data_from_raw_card(raw_card: dict) -> dict[str, bool]: def preprocess_card(card: dict[str, Any]) -> list[dict[str, Any]]: # noqa: PLR0915,C901,PLR0912 """Preprocess a card to remove invalid cards and add necessary fields. - For Double-Faced Cards (DFCs), returns multiple dictionaries (one per face). - For single-faced cards, returns a list with one dictionary. + A multi-face card (transform, MDFC, split, adventure, flip) is merged into ONE row + carrying the front face's identity and every face's searchable data — see + `_merge_processed_faces`. Single-faced cards return a list with one dictionary. Returns an empty list for invalid/filtered cards. """ if not set(card["legalities"].values()) & {"legal", "restricted"}: @@ -154,21 +291,41 @@ def preprocess_card(card: dict[str, Any]) -> list[dict[str, Any]]: # noqa: PLR0 # Recursive case: processing a face card["face_name"] = card.get("name") - # Handle cards with card_faces (DFCs) + # Handle cards with card_faces (DFCs): process each face through the full pipeline below, + # then collapse the per-face rows into the card's one searchable row. card_faces = card.get("card_faces") if card_faces: for creature_attribute in ["creature_power", "creature_toughness"]: card.pop(creature_attribute, None) card.pop(f"{creature_attribute}_text", None) - processed_faces = [] - for face_idx, face_data in enumerate(card_faces, start=1): + face_rows = [] + for face_data in card_faces: # Merge card-level data with face-specific data - # Precedence: face_idx override > face_data (name, type_line, etc.) > card (legalities, games, etc.) - merged = copy.deepcopy(card) | face_data | {"face_idx": face_idx} + # Precedence: face_data (name, type_line, etc.) > card (legalities, games, etc.) + merged = copy.deepcopy(card) | face_data merged.pop("card_faces", None) # Don't keep recursing - processed_faces_for_face = preprocess_card(merged) - processed_faces.extend(processed_faces_for_face) - return processed_faces + face_rows.extend(preprocess_card(merged)) + if not face_rows: + return [] + merged_row = _merge_processed_faces(face_rows) + # The blob is the card-level object with its faces re-attached — what Scryfall sent, not a + # face promoted to look like a card. Every searchable field is already merged onto the row + # above, so the blob has no derivation left to do, and keeping it verbatim is what makes it + # answerable: a card object cannot be rebuilt from a face (`card_faces` is gone, `name` and + # `type_line` are the front's, and which fields a real card carries at top level varies by + # layout — a split card has `mana_cost` and `image_uris` there, a transform card does not). + # + # The one consumer that read a *face* field off the blob is `image_uris`, which for a + # transform card now lives only under `card_faces`; every reader coalesces to + # `card_faces->0` (scripts/copy_images_to_s3.py, scripts/prefer_weights.py). Everything else + # read from the blob — lang, set_type, games, finishes, frame_effects, image_status, + # reserved, game_changer — is card-level and identical either way. + merged_row["raw_card_blob"] = copy.deepcopy(card) | {"card_faces": card_faces} + # The engine's copy of the same thing. raw_card_blob is a Postgres column and the SQL path + # is a fallback, so anything only the blob carries is unanswerable on the engine path. + merged_row["card_faces"] = _face_records(card_faces) + merged_row["card_compat_blob"] = _compat_blob(card) + return [merged_row] # Single face case - set defaults card.setdefault("face_name", card.get("name")) @@ -184,6 +341,7 @@ def preprocess_card(card: dict[str, Any]) -> list[dict[str, Any]]: # noqa: PLR0 # Store the original card data before modifications for raw_card_blob raw_card_data = copy.deepcopy(card) card["raw_card_blob"] = raw_card_data + card["card_compat_blob"] = _compat_blob(raw_card_data) card["scryfall_id"] = card["id"] card_types, card_subtypes = parse_type_line(card["type_line"]) diff --git a/api/db/2026-08-09-01-scryfall-cards-api.sql b/api/db/2026-08-09-01-scryfall-cards-api.sql new file mode 100644 index 000000000..d247f88e7 --- /dev/null +++ b/api/db/2026-08-09-01-scryfall-cards-api.sql @@ -0,0 +1,84 @@ +-- Backing store for the Scryfall-compatible /cards/* API (api/scryfall_compat/). +-- +-- Two parts, neither of which any existing route reads: +-- +-- 1. Lookup indexes for the identifiers Scryfall routes by that live inside raw_card_blob rather +-- than in a column of their own. Each partial index predicates on the *same* expression the +-- lookup compares, so the planner can prove the query implies the predicate. +-- +-- 2. magic.rulings, backing /cards/:id/rulings and its four sibling routes. +-- +-- No column holds the card object the API serves: raw_card_blob is already what Scryfall sent, so +-- api/scryfall_compat/objects.py recovers the card from it by stripping the three keys the importer +-- adds. That is true of a multi-face printing only as of the merged-row work; see the deployment +-- note in docs/issues/local-scryfall-cards-api.md. + + +-- /cards/:code/:number and /cards/:code/:number/:lang. On lower(card_set_code) rather than the +-- column, because Scryfall's set codes are case-insensitive and the lookup folds the segment; the +-- corpus happens to store them lowercase already, but nothing constrains it to. +CREATE INDEX IF NOT EXISTS idx_cards_set_collector_lang + ON magic.cards USING btree (lower(card_set_code), collector_number, (raw_card_blob ->> 'lang')); + +-- GET /cards/named?exact=. card_name_folded is already lowercase (fold_accents() lowercases before +-- folding), so lower() here is a no-op on the value and only exists so the indexed expression is +-- the one the lookup writes. The two split_part indexes cover the face names of a "Front // Back" +-- card, which Scryfall's exact match accepts alongside the combined name: with all three indexed, +-- the lookup's OR is a BitmapOr rather than a sequential scan. +CREATE INDEX IF NOT EXISTS idx_cards_name_folded_exact + ON magic.cards USING btree (lower(card_name_folded)); +CREATE INDEX IF NOT EXISTS idx_cards_name_folded_front_face + ON magic.cards USING btree (lower(split_part(card_name_folded, ' // ', 1))); +CREATE INDEX IF NOT EXISTS idx_cards_name_folded_back_face + ON magic.cards USING btree (lower(split_part(card_name_folded, ' // ', 2))); + +-- /cards/multiverse/:id -- multiverse_ids is an array, so containment rather than equality. +CREATE INDEX IF NOT EXISTS idx_cards_multiverse_ids + ON magic.cards USING gin ((raw_card_blob -> 'multiverse_ids') jsonb_path_ops); + +-- /cards/mtgo/:id matches either the regular or the foil MTGO id, so both are indexed. +CREATE INDEX IF NOT EXISTS idx_cards_mtgo_id + ON magic.cards USING btree (((raw_card_blob ->> 'mtgo_id')::bigint)) + WHERE ((raw_card_blob ->> 'mtgo_id')::bigint) IS NOT NULL; + +CREATE INDEX IF NOT EXISTS idx_cards_mtgo_foil_id + ON magic.cards USING btree (((raw_card_blob ->> 'mtgo_foil_id')::bigint)) + WHERE ((raw_card_blob ->> 'mtgo_foil_id')::bigint) IS NOT NULL; + +-- /cards/arena/:id +CREATE INDEX IF NOT EXISTS idx_cards_arena_id + ON magic.cards USING btree (((raw_card_blob ->> 'arena_id')::bigint)) + WHERE ((raw_card_blob ->> 'arena_id')::bigint) IS NOT NULL; + +-- /cards/tcgplayer/:id matches either the regular or the etched TCGplayer id. +CREATE INDEX IF NOT EXISTS idx_cards_tcgplayer_id + ON magic.cards USING btree (((raw_card_blob ->> 'tcgplayer_id')::bigint)) + WHERE ((raw_card_blob ->> 'tcgplayer_id')::bigint) IS NOT NULL; + +CREATE INDEX IF NOT EXISTS idx_cards_tcgplayer_etched_id + ON magic.cards USING btree (((raw_card_blob ->> 'tcgplayer_etched_id')::bigint)) + WHERE ((raw_card_blob ->> 'tcgplayer_etched_id')::bigint) IS NOT NULL; + +-- /cards/cardmarket/:id +CREATE INDEX IF NOT EXISTS idx_cards_cardmarket_id + ON magic.cards USING btree (((raw_card_blob ->> 'cardmarket_id')::bigint)) + WHERE ((raw_card_blob ->> 'cardmarket_id')::bigint) IS NOT NULL; + +-- The oracle_id and illustration_id identifiers POST /cards/collection accepts, and the oracle_id +-- the rulings hang off, are already served by idx_cards_oracle_id and idx_cards_illustration_id +-- from 2026-06-21-01-bulk-tag-import.sql. + +CREATE TABLE IF NOT EXISTS magic.rulings ( + oracle_id uuid NOT NULL, + source text NOT NULL, + published_at date NOT NULL, + comment text NOT NULL +); + +COMMENT ON TABLE magic.rulings IS + 'Scryfall rulings bulk data, keyed by oracle_id. One row per ruling; a card has zero or more.'; + +-- The bulk file carries no ruling id, so identity is the tuple itself. md5() rather than the raw +-- comment because a btree entry is capped at ~2700 bytes and rulings run longer than that. +CREATE UNIQUE INDEX IF NOT EXISTS idx_rulings_identity + ON magic.rulings USING btree (oracle_id, source, published_at, md5(comment)); diff --git a/api/db/2026-08-10-01-engine-card-objects.sql b/api/db/2026-08-10-01-engine-card-objects.sql new file mode 100644 index 000000000..eba384ee7 --- /dev/null +++ b/api/db/2026-08-10-01-engine-card-objects.sql @@ -0,0 +1,64 @@ +-- The engine serves searches; Postgres is the fallback for when the engine errors. Anything only +-- raw_card_blob carries is therefore unanswerable on the primary path -- a jsonb column is not in +-- the store, and the store is all an engine request reads. That is why /cards/* was SQL-only. +-- +-- Two columns fix it, both shaped like the jsonb grab-bags this table already has (card_frame_data, +-- card_is_tags, card_legalities): +-- +-- card_compat_blob -- every Scryfall key that is neither stored in a column of its own nor a pure +-- function of one. Measured on a real card: 680 bytes against raw_card_blob's +-- 5,881, because the blob is overwhelmingly redundant with columns we already +-- have plus URLs derivable from the card's id. +-- card_faces -- each face's own fields, so a face is a thing the engine can read rather +-- than something recoverable only by re-parsing a blob. +-- +-- Both backfill from raw_card_blob, so this deploys without a reimport (same reasoning as +-- 2026-08-06-01-lowercase-keywords.sql: the bulk import would rewrite these anyway, but the +-- query-side change should not wait for a window where /cards/* returns nothing). +ALTER TABLE magic.cards ADD COLUMN IF NOT EXISTS card_compat_blob jsonb; +ALTER TABLE magic.cards ADD COLUMN IF NOT EXISTS card_faces jsonb; + +-- Subtractive rather than enumerated: the residue is defined as "what is left once everything we +-- store or can derive is removed", so a Scryfall key we have never seen lands here by default +-- instead of being silently dropped. Keys removed below are, in order: stored in their own column; +-- derivable from id/set/collector_number/oracle_id; the faces, which get their own column; and the +-- keys preprocess_card adds to the object before it is snapshotted. +-- +-- `prices` is deliberately NOT removed even though price_usd/eur/tix are stored: usd_foil, +-- usd_etched and eur_foil are not, and keeping the object whole costs a few bytes against losing +-- three fields. +UPDATE magic.cards +SET card_compat_blob = raw_card_blob - ARRAY[ + 'id', 'oracle_id', 'name', 'released_at', 'layout', 'mana_cost', 'cmc', 'type_line', + 'oracle_text', 'power', 'toughness', 'loyalty', 'colors', 'color_identity', 'keywords', + 'set', 'set_name', 'collector_number', 'rarity', 'flavor_text', 'artist', + 'illustration_id', 'border_color', 'edhrec_rank', 'legalities', 'produced_mana', + 'watermark', 'reserved', 'game_changer', 'frame', + 'object', 'uri', 'scryfall_uri', 'image_uris', 'rulings_uri', 'prints_search_uri', + 'set_uri', 'set_search_uri', 'scryfall_set_uri', 'card_back_id', 'related_uris', + 'purchase_uris', 'resource_id', + 'card_faces', + 'card_name', 'face_name', 'face_idx', 'scryfall_id' + ], + card_faces = CASE + WHEN raw_card_blob ? 'card_faces' THEN ( + SELECT jsonb_agg( + face - ARRAY['object', 'image_uris'] + ORDER BY ordinality + ) + FROM jsonb_array_elements(raw_card_blob -> 'card_faces') WITH ORDINALITY AS t(face, ordinality) + ) + ELSE NULL + END +WHERE card_compat_blob IS NULL; + +-- The engine reload reads every ENGINE_COLUMNS value for every row, so a NULL here would mean a +-- per-row branch in the hot path for a case that cannot legitimately occur. +ALTER TABLE magic.cards ALTER COLUMN card_compat_blob SET DEFAULT '{}'::jsonb; +UPDATE magic.cards SET card_compat_blob = '{}'::jsonb WHERE card_compat_blob IS NULL; +ALTER TABLE magic.cards ALTER COLUMN card_compat_blob SET NOT NULL; + +ALTER TABLE magic.cards ADD CONSTRAINT card_compat_blob_must_be_object + CHECK ((jsonb_typeof(card_compat_blob) = 'object'::text)); +ALTER TABLE magic.cards ADD CONSTRAINT card_faces_must_be_array + CHECK (((card_faces IS NULL) OR (jsonb_typeof(card_faces) = 'array'::text))); diff --git a/api/db/2026-08-11-02-scryfall-sets-catalogs-symbology.sql b/api/db/2026-08-11-02-scryfall-sets-catalogs-symbology.sql new file mode 100644 index 000000000..b281659c4 --- /dev/null +++ b/api/db/2026-08-11-02-scryfall-sets-catalogs-symbology.sql @@ -0,0 +1,82 @@ +-- Backing store for the reference half of the Scryfall API: /sets, /catalog/* and /symbology. +-- +-- All three are mirrored from api.scryfall.com rather than derived from magic.cards, and the reason +-- is the same in each case: what the corpus can prove is a strict subset of what these endpoints +-- have to say. +-- +-- * A Set object carries eight fields no card carries -- tcgplayer_id, mtgo_code, arena_code, +-- icon_svg_uri, block, block_code, parent_set_code, printed_size. /sets/tcgplayer/:id cannot be +-- answered at all without the first of them. +-- * card_count is Scryfall's count for the whole set, not the count this instance imported. The +-- corpus is a filtered subset (no tokens, funny sets or digital-only printings), so deriving it +-- would report a number no other Scryfall client agrees with. +-- * A card symbol's svg_uri and gatherer_alternates exist nowhere in the card data. +-- +-- Each table therefore stores the upstream object whole, in jsonb, and lifts out only what a lookup +-- keys on. That keeps the served bytes identical to Scryfall's without a column-by-column mapping +-- that would silently drop a field Scryfall adds later. + + +-- One row per set, the object exactly as Scryfall sent it. +-- +-- `position` preserves the order /sets returns rather than recomputing one. The list is sorted by +-- released_at descending, but sets sharing a release date come back in an order that is neither +-- alphabetical by code nor by name (2026-11-13 yields trk, trc, ttrk, sds), and nothing in the +-- object reproduces it. Storing the index is exact and costs four bytes. +CREATE TABLE IF NOT EXISTS magic.sets ( + id uuid PRIMARY KEY, + code text NOT NULL, + tcgplayer_id bigint, + position integer NOT NULL, + set_object jsonb NOT NULL +); + +COMMENT ON TABLE magic.sets IS + 'Scryfall Set objects, mirrored from api.scryfall.com/sets. One row per set; set_object is the upstream object verbatim.'; +COMMENT ON COLUMN magic.sets.position IS + 'Index in Scryfall''s own /sets ordering, which is not reproducible from the object.'; + +-- /sets/:code. Scryfall matches a set code case-insensitively, so the index is on the folded value +-- and the lookup folds its segment to match. Unique because a code identifies one set. +CREATE UNIQUE INDEX IF NOT EXISTS idx_sets_code ON magic.sets USING btree (lower(code)); + +-- /sets/tcgplayer/:id. Partial because most sets have no TCGplayer id, and a null there is not an +-- identifier anyone can look up. +CREATE INDEX IF NOT EXISTS idx_sets_tcgplayer_id + ON magic.sets USING btree (tcgplayer_id) + WHERE tcgplayer_id IS NOT NULL; + +-- GET /sets returns every row in one List object, so the listing is an index-only ordering scan. +CREATE INDEX IF NOT EXISTS idx_sets_position ON magic.sets USING btree (position); + + +-- The twenty /catalog/* endpoints, one row each. +-- +-- A table of (name, entries) rather than twenty tables or twenty columns: the endpoints differ only +-- in which list of strings they return, the set of them is fixed by Scryfall rather than by this +-- schema, and a catalog is only ever read or replaced whole. `entries` is a jsonb array of strings, +-- which is what the Catalog object's `data` is -- named `entries` rather than the obvious `values` +-- because VALUES is a reserved word and the column would need quoting at every use site. +CREATE TABLE IF NOT EXISTS magic.catalogs ( + name text PRIMARY KEY, + entries jsonb NOT NULL +); + +COMMENT ON TABLE magic.catalogs IS + 'Scryfall Catalog payloads, one row per /catalog/:name endpoint, entries being the data array verbatim.'; + + +-- /symbology. Eighty-odd rows, each the upstream card_symbol object. +-- +-- `position` for the same reason magic.sets has one: Scryfall returns the symbols in a fixed order +-- that no field on the object reproduces, and /symbology is served as that list. +CREATE TABLE IF NOT EXISTS magic.card_symbols ( + symbol text PRIMARY KEY, + position integer NOT NULL, + symbol_object jsonb NOT NULL +); + +COMMENT ON TABLE magic.card_symbols IS + 'Scryfall CardSymbol objects, mirrored from api.scryfall.com/symbology, in the order that endpoint returns them.'; + +CREATE INDEX IF NOT EXISTS idx_card_symbols_position ON magic.card_symbols USING btree (position); diff --git a/api/enums.py b/api/enums.py index f970591e8..4f64cc7c7 100644 --- a/api/enums.py +++ b/api/enums.py @@ -23,14 +23,28 @@ class PreferOrder(enum.StrEnum): class CardOrdering(enum.StrEnum): - """Enum for the ordering of the cards.""" + """Enum for the ordering of the cards. + Every member must have a `SortCol` arm in card_engine/src/lib.rs and a `sql_orderby` entry in + api_resource.py. `orderby_to_col` falls through to edhrec on an unknown name, so a member added + here and nowhere else makes the engine and SQL paths sort the same query differently. + + `cubecobra` is this project's own; the rest are Scryfall's `order=` vocabulary. Scryfall's + `penny` and `review` are deliberately absent -- see docs/issues/local-engine-order-vocabulary.md. + """ + + ARTIST = enum.auto() CMC = enum.auto() + COLOR = enum.auto() CUBECOBRA = enum.auto() EDHREC = enum.auto() + EUR = enum.auto() NAME = enum.auto() POWER = enum.auto() RARITY = enum.auto() + RELEASED = enum.auto() + SET = enum.auto() + TIX = enum.auto() TOUGHNESS = enum.auto() USD = enum.auto() @@ -43,7 +57,43 @@ class ResponseShape(enum.StrEnum): class SortDirection(enum.StrEnum): - """Enum for the direction of the sort.""" + """Enum for the direction of the sort. + + AUTO is resolved to ASC or DESC per ordering before any search path sees it (see + AUTO_DIRECTIONS and `resolve_direction`), so neither the engine nor the SQL builder ever + receives it. + """ ASC = enum.auto() DESC = enum.auto() + AUTO = enum.auto() + + +# What `dir=auto` means for each ordering, measured against api.scryfall.com on 2026-08-09 by +# comparing the `auto` page against the `asc` and `desc` pages of the same query. Only these five +# invert; every other ordering, edhrec included, resolves ascending -- for edhrec that is the +# direction putting rank 1 first, so "most popular first" and "ascending rank" are the same thing. +AUTO_DESCENDING_ORDERINGS: frozenset[CardOrdering] = frozenset( + { + CardOrdering.RELEASED, + CardOrdering.RARITY, + CardOrdering.USD, + CardOrdering.TIX, + CardOrdering.EUR, + }, +) + + +def resolve_direction(direction: SortDirection, orderby: CardOrdering) -> SortDirection: + """Resolve AUTO against an ordering, leaving an explicit direction alone. + + Args: + direction: The requested direction, possibly AUTO. + orderby: The ordering AUTO is being resolved against. + + Returns: + ASC or DESC, never AUTO. + """ + if direction is not SortDirection.AUTO: + return direction + return SortDirection.DESC if orderby in AUTO_DESCENDING_ORDERINGS else SortDirection.ASC diff --git a/api/parsing/db_info.py b/api/parsing/db_info.py index 7f86df7d4..7131d7892 100644 --- a/api/parsing/db_info.py +++ b/api/parsing/db_info.py @@ -320,6 +320,7 @@ def __repr__(self: FieldInfo) -> str: CARD_TYPES = { "Artifact", + "Battle", # reaches the corpus once faces merge (#400): every battle is a transform front "Conspiracy", "Creature", "Enchantment", diff --git a/api/parsing/tests/test_pyparsing_parser.py b/api/parsing/tests/test_pyparsing_parser.py index 6e2ab99aa..b82319327 100644 --- a/api/parsing/tests/test_pyparsing_parser.py +++ b/api/parsing/tests/test_pyparsing_parser.py @@ -1258,3 +1258,24 @@ def test_hyphenated_words_edge_cases_fail(invalid_query: str) -> None: """ with pytest.raises(ValueError, match="Failed to parse query"): hand_parser.parse_str_to_query(invalid_query) + + +class TestBattleTypeRouting: + """`t:battle` routes to card_types, matching every other card type. + + Battle was absent from CARD_TYPES, so it fell through to the subtype arm on both the + SQL and engine paths — invisible while the corpus stored every battle as its back face + (#400), a guaranteed zero-match once the face merge put Battle into card_types. + """ + + def test_battle_routes_like_a_card_type(self) -> None: + """t:battle generates card_types SQL binding ['Battle'], as t:creature does for its type.""" + sql, params = parsing.generate_sql_query(parsing.parse_scryfall_query("t:battle")) + assert "card_types" in sql + assert "card_subtypes" not in sql + assert ["Battle"] in params.values() + + def test_siege_still_routes_as_a_subtype(self) -> None: + """t:siege stays on the subtype arm; only the card type moved.""" + siege_sql = str(parsing.generate_sql_query(parsing.parse_scryfall_query("t:siege"))) + assert "card_subtypes" in siege_sql diff --git a/api/rulings_import.py b/api/rulings_import.py new file mode 100644 index 000000000..050129d2f --- /dev/null +++ b/api/rulings_import.py @@ -0,0 +1,103 @@ +"""Import Scryfall rulings bulk data into `magic.rulings`. + +Rulings back `/cards/:id/rulings` and its four sibling routes. They hang off `oracle_id`, not off a +printing, so the table is independent of `magic.cards` and is loaded the same way the tag +collections are: streamed from the cached bulk file and written in batches. + +The load is a whole-table replace inside one transaction, rather than an upsert. The bulk file +carries no ruling id — a row's identity is the tuple itself — and rulings are occasionally +retracted, so "insert what is there" would accumulate rows that Scryfall has withdrawn. A replace +also cannot get the pruning wrong when one card's rulings straddle a batch boundary, which is the +failure an incremental prune invites. `DELETE` rather than `TRUNCATE` so readers keep seeing the +previous contents through MVCC instead of blocking on an ACCESS EXCLUSIVE lock for the load. +""" + +from __future__ import annotations + +import itertools +import logging +import time +from typing import TYPE_CHECKING, Any + +from psycopg.types.json import Jsonb + +from api.scryfall_bulk_data_fetcher import BulkDataKey + +if TYPE_CHECKING: + from collections.abc import Iterable, Iterator + + import psycopg_pool + + from api.scryfall_bulk_data_fetcher import ScryfallBulkDataFetcher + +logger = logging.getLogger(__name__) + +# Rows per INSERT. The whole batch travels as one jsonb bind parameter, so this sets the statement's +# server-side peak; the rulings file is two orders of magnitude smaller than the cards file, so it +# can sit well above the card upsert's page size and still be a fraction of its parameter size. +_BATCH_SIZE = 5_000 + +_INSERT_SQL = """ +INSERT INTO magic.rulings (oracle_id, source, published_at, comment) +SELECT + (entry ->> 'oracle_id')::uuid, + entry ->> 'source', + (entry ->> 'published_at')::date, + entry ->> 'comment' +FROM jsonb_array_elements(%(rows)s) AS entry +ON CONFLICT DO NOTHING +""" + +_REQUIRED_FIELDS = ("oracle_id", "source", "published_at", "comment") + +# Scryfall publishes `published_at` as a bare date; slicing rather than parsing keeps a future +# timestamp form from failing the ::date cast. +_DATE_LENGTH = 10 + + +def _valid_rulings(rulings: Iterable[dict[str, Any]]) -> Iterator[dict[str, Any]]: + """Yield the entries that carry every column the table requires. + + Args: + rulings: Raw entries from the bulk file. + + Yields: + One normalized row per usable entry. + """ + for ruling in rulings: + if all(ruling.get(field) for field in _REQUIRED_FIELDS): + yield { + "oracle_id": ruling["oracle_id"], + "source": ruling["source"], + "published_at": str(ruling["published_at"])[:_DATE_LENGTH], + "comment": ruling["comment"], + } + + +def import_rulings(conn_pool: psycopg_pool.ConnectionPool, fetcher: ScryfallBulkDataFetcher) -> int: + """Replace `magic.rulings` with the current rulings bulk file. + + Args: + conn_pool: Pool to run the load through. + fetcher: Bulk data fetcher, which caches the download between runs. + + Returns: + The number of rulings loaded. + """ + before = time.monotonic() + loaded = 0 + with conn_pool.connection() as conn: + # One transaction: the DELETE is only visible to other sessions once the reload commits, + # so no request can observe an empty rulings table. + with conn.cursor() as cursor: + cursor.execute("DELETE FROM magic.rulings") + for batch in itertools.batched(_valid_rulings(fetcher.stream_data_for_key(BulkDataKey.RULINGS)), _BATCH_SIZE): + cursor.execute(_INSERT_SQL, {"rows": Jsonb(list(batch))}) + # rowcount, not len(batch): the file repeats a tuple often enough to matter -- 37 of + # 77,998 entries on 2026-08-11 -- and ON CONFLICT DO NOTHING drops those. Counting + # what was sent would report a row total the table does not hold. + loaded += cursor.rowcount + conn.commit() + + logger.info("Imported %d rulings in %.2f seconds", loaded, time.monotonic() - before) + return loaded diff --git a/api/scryfall_bulk_data_fetcher.py b/api/scryfall_bulk_data_fetcher.py index c12d0c44c..da35d75bc 100644 --- a/api/scryfall_bulk_data_fetcher.py +++ b/api/scryfall_bulk_data_fetcher.py @@ -157,6 +157,27 @@ def _get(self, url: str, *, timeout: int, **kwargs: object) -> requests.Response response.raise_for_status() return response + def fetch_api_json(self, path: str, *, timeout: int = 30) -> dict: + """GET one Scryfall API endpoint and decode it. + + The reference data — sets, catalogs, symbology — is published as ordinary API responses + rather than as bulk dumps, and is small enough to fetch whole. It goes through this session + so it inherits the same retry policy and error logging the dumps get, rather than opening a + second HTTP client with its own behaviour. + + Args: + path: Path below the API root, with no leading slash (e.g. "catalog/creature-types"). + timeout: Per-attempt request timeout in seconds. + + Returns: + The decoded response body. + + Raises: + requests.HTTPError: If the final response after retries is not 2xx. + requests.RequestException: If the request fails at the transport level. + """ + return self._get(f"https://api.scryfall.com/{path}", timeout=timeout).json() + @cachebox_cached(cache=TTLCache(maxsize=2, global_ttl=5 * MINUTE)) def list_bulk_data(self) -> dict[BulkDataKey, dict]: """Fetch bulk data from Scryfall, ignoring bulk data types we don't recognize.""" diff --git a/api/scryfall_compat/__init__.py b/api/scryfall_compat/__init__.py new file mode 100644 index 000000000..162935cf7 --- /dev/null +++ b/api/scryfall_compat/__init__.py @@ -0,0 +1,12 @@ +"""Scryfall-compatible `/cards/*` API. + +The routes in `routes.py` answer the same paths as api.scryfall.com with the same parameters and +the same response objects, so a client can be pointed at this host by swapping its base URL. +`objects.py` holds the payload construction — card reconstruction, List/Catalog/error envelopes, +and the `format=text` rendering — with no knowledge of Falcon or of the database. +""" + +from api.scryfall_compat.reference_routes import ScryfallReferenceRoutes +from api.scryfall_compat.routes import ScryfallCardsRoutes + +__all__ = ["ScryfallCardsRoutes", "ScryfallReferenceRoutes"] diff --git a/api/scryfall_compat/fixtures/card_object_parity.json b/api/scryfall_compat/fixtures/card_object_parity.json new file mode 100644 index 000000000..570a73e5b --- /dev/null +++ b/api/scryfall_compat/fixtures/card_object_parity.json @@ -0,0 +1,1175 @@ +{ + "_comment": [ + "CROSS-LANGUAGE PARITY FIXTURE for the Scryfall card object.", + "", + "The same card JSON is built twice: card_engine/src/card_object.rs serves the engine path and", + "api/scryfall_compat/objects.py serves the SQL path. Both answer /cards/*, so a difference", + "between them is a difference a client can see, and nothing else compares them.", + "", + "Each case is an engine row plus the object BOTH implementations produce from it, built with", + "base_url 'https://api.example/v1'. card_object.rs asserts it in Rust and", + "test_scryfall_compat_objects.py asserts it in Python, so either one drifting turns its own", + "CI job red.", + "", + "Values and key PRESENCE are what this pins; key ORDER is not (both sides are compared as", + "parsed objects). Scryfall's key order is pinned separately by the position assertions in", + "card_object.rs, which is the only place the wire order is observable.", + "", + "To add a case: add the row, run both suites, and paste the object they agree on. If they do", + "not agree, that is the bug this file exists to find." + ], + "base_url": "https://api.example/v1", + "cases": [ + { + "case": "a plain single-faced card", + "row": { + "scryfall_id": "01000000-0000-0000-0000-000000000001", + "oracle_id": "11111111-2222-3333-4444-555555555555", + "name": "Lightning Bolt", + "mana_cost": "{R}", + "cmc": 1, + "type_line": "Instant", + "oracle_text": "Lightning Bolt deals 3 damage to any target.", + "colors": [ + "R" + ], + "color_identity": [ + "R" + ], + "set_code": "lea", + "set_name": "Limited Edition Alpha", + "collector_number": "161", + "rarity": "common", + "lang": "en", + "layout": "normal", + "artist": "Christopher Rush", + "released_at": "1993-08-05", + "multiverse_ids": [ + 209 + ], + "games": [ + "paper" + ], + "tcgplayer_id": 1234 + }, + "expected": { + "artist": "Christopher Rush", + "booster": false, + "border_color": null, + "card_back_id": "0aeebaf5-8c7d-4636-9e82-8c27447861f7", + "cmc": 1.0, + "collector_number": "161", + "color_identity": [ + "R" + ], + "colors": [ + "R" + ], + "digital": false, + "finishes": [], + "full_art": false, + "games": [ + "paper" + ], + "highres_image": false, + "id": "01000000-0000-0000-0000-000000000001", + "illustration_id": null, + "image_status": null, + "image_uris": { + "art": "https://cards.scryfall.io/art/front/0/1/01000000-0000-0000-0000-000000000001.webp", + "art_crop": "https://cards.scryfall.io/art_crop/front/0/1/01000000-0000-0000-0000-000000000001.jpg", + "border_crop": "https://cards.scryfall.io/border_crop/front/0/1/01000000-0000-0000-0000-000000000001.jpg", + "crop": "https://cards.scryfall.io/crop/front/0/1/01000000-0000-0000-0000-000000000001.webp", + "display": "https://cards.scryfall.io/display/front/0/1/01000000-0000-0000-0000-000000000001.webp", + "grid": "https://cards.scryfall.io/grid/front/0/1/01000000-0000-0000-0000-000000000001.webp", + "large": "https://cards.scryfall.io/large/front/0/1/01000000-0000-0000-0000-000000000001.jpg", + "normal": "https://cards.scryfall.io/normal/front/0/1/01000000-0000-0000-0000-000000000001.jpg", + "png": "https://cards.scryfall.io/png/front/0/1/01000000-0000-0000-0000-000000000001.png", + "small": "https://cards.scryfall.io/small/front/0/1/01000000-0000-0000-0000-000000000001.jpg", + "thumb": "https://cards.scryfall.io/thumb/front/0/1/01000000-0000-0000-0000-000000000001.webp" + }, + "keywords": [], + "lang": "en", + "layout": "normal", + "mana_cost": "{R}", + "multiverse_ids": [ + 209 + ], + "name": "Lightning Bolt", + "object": "card", + "oracle_id": "11111111-2222-3333-4444-555555555555", + "oracle_text": "Lightning Bolt deals 3 damage to any target.", + "oversized": false, + "prices": { + "eur": null, + "eur_foil": null, + "tix": null, + "usd": null, + "usd_etched": null, + "usd_foil": null + }, + "prints_search_uri": "https://api.example/v1/cards/search?order=released&q=oracleid%3A11111111-2222-3333-4444-555555555555&unique=prints", + "promo": false, + "purchase_uris": { + "cardhoarder": "https://www.cardhoarder.com/cards?data%5Bsearch%5D=Lightning+Bolt", + "cardmarket": "https://www.cardmarket.com/en/Magic/Products/Search?searchString=Lightning+Bolt", + "tcgplayer": "https://www.tcgplayer.com/product/1234?page=1" + }, + "rarity": "common", + "related_uris": { + "edhrec": "https://edhrec.com/route/?cc=Lightning+Bolt", + "gatherer": "https://gatherer.wizards.com/Pages/Card/Details.aspx?multiverseid=209&printed=false", + "tcgplayer_infinite_articles": "https://www.tcgplayer.com/search/articles?productLineName=magic&q=Lightning+Bolt", + "tcgplayer_infinite_decks": "https://www.tcgplayer.com/search/decks?productLineName=magic&q=Lightning+Bolt" + }, + "released_at": "1993-08-05", + "reprint": false, + "reserved": false, + "rulings_uri": "https://api.example/v1/cards/01000000-0000-0000-0000-000000000001/rulings", + "scryfall_set_uri": "https://scryfall.com/sets/lea?utm_source=api", + "scryfall_uri": "https://scryfall.com/card/lea/161/lightning-bolt?utm_source=api", + "set": "lea", + "set_id": null, + "set_name": "Limited Edition Alpha", + "set_search_uri": "https://api.example/v1/cards/search?order=set&q=e%3Alea&unique=prints", + "set_type": null, + "set_uri": null, + "story_spotlight": false, + "tcgplayer_id": 1234, + "textless": false, + "type_line": "Instant", + "uri": "https://api.example/v1/cards/01000000-0000-0000-0000-000000000001", + "variation": false + } + }, + { + "case": "a slug that deletes rather than hyphenates", + "row": { + "scryfall_id": "01000000-0000-0000-0000-000000000002", + "name": "Erayo's Essence", + "set_code": "chk", + "collector_number": "1", + "lang": "en", + "layout": "normal", + "games": [ + "paper" + ] + }, + "expected": { + "artist": null, + "booster": false, + "border_color": null, + "card_back_id": "0aeebaf5-8c7d-4636-9e82-8c27447861f7", + "cmc": null, + "collector_number": "1", + "color_identity": [], + "colors": [], + "digital": false, + "finishes": [], + "full_art": false, + "games": [ + "paper" + ], + "highres_image": false, + "id": "01000000-0000-0000-0000-000000000002", + "illustration_id": null, + "image_status": null, + "image_uris": { + "art": "https://cards.scryfall.io/art/front/0/1/01000000-0000-0000-0000-000000000002.webp", + "art_crop": "https://cards.scryfall.io/art_crop/front/0/1/01000000-0000-0000-0000-000000000002.jpg", + "border_crop": "https://cards.scryfall.io/border_crop/front/0/1/01000000-0000-0000-0000-000000000002.jpg", + "crop": "https://cards.scryfall.io/crop/front/0/1/01000000-0000-0000-0000-000000000002.webp", + "display": "https://cards.scryfall.io/display/front/0/1/01000000-0000-0000-0000-000000000002.webp", + "grid": "https://cards.scryfall.io/grid/front/0/1/01000000-0000-0000-0000-000000000002.webp", + "large": "https://cards.scryfall.io/large/front/0/1/01000000-0000-0000-0000-000000000002.jpg", + "normal": "https://cards.scryfall.io/normal/front/0/1/01000000-0000-0000-0000-000000000002.jpg", + "png": "https://cards.scryfall.io/png/front/0/1/01000000-0000-0000-0000-000000000002.png", + "small": "https://cards.scryfall.io/small/front/0/1/01000000-0000-0000-0000-000000000002.jpg", + "thumb": "https://cards.scryfall.io/thumb/front/0/1/01000000-0000-0000-0000-000000000002.webp" + }, + "keywords": [], + "lang": "en", + "layout": "normal", + "mana_cost": null, + "multiverse_ids": [], + "name": "Erayo's Essence", + "object": "card", + "oracle_id": "", + "oracle_text": null, + "oversized": false, + "prices": { + "eur": null, + "eur_foil": null, + "tix": null, + "usd": null, + "usd_etched": null, + "usd_foil": null + }, + "prints_search_uri": "https://api.example/v1/cards/search?order=released&q=oracleid%3A&unique=prints", + "promo": false, + "purchase_uris": { + "cardhoarder": "https://www.cardhoarder.com/cards?data%5Bsearch%5D=Erayo%27s+Essence", + "cardmarket": "https://www.cardmarket.com/en/Magic/Products/Search?searchString=Erayo%27s+Essence", + "tcgplayer": "https://www.tcgplayer.com/search/magic/product?productLineName=magic&q=Erayo%27s+Essence&view=grid" + }, + "rarity": null, + "related_uris": { + "edhrec": "https://edhrec.com/route/?cc=Erayo%27s+Essence", + "tcgplayer_infinite_articles": "https://www.tcgplayer.com/search/articles?productLineName=magic&q=Erayo%27s+Essence", + "tcgplayer_infinite_decks": "https://www.tcgplayer.com/search/decks?productLineName=magic&q=Erayo%27s+Essence" + }, + "released_at": null, + "reprint": false, + "reserved": false, + "rulings_uri": "https://api.example/v1/cards/01000000-0000-0000-0000-000000000002/rulings", + "scryfall_set_uri": "https://scryfall.com/sets/chk?utm_source=api", + "scryfall_uri": "https://scryfall.com/card/chk/1/erayos-essence?utm_source=api", + "set": "chk", + "set_id": null, + "set_name": null, + "set_search_uri": "https://api.example/v1/cards/search?order=set&q=e%3Achk&unique=prints", + "set_type": null, + "set_uri": null, + "story_spotlight": false, + "textless": false, + "type_line": null, + "uri": "https://api.example/v1/cards/01000000-0000-0000-0000-000000000002", + "variation": false + } + }, + { + "case": "a foreign printing takes the language path segment", + "row": { + "scryfall_id": "01000000-0000-0000-0000-000000000003", + "name": "Unmoored Ego", + "set_code": "grn", + "collector_number": "212", + "lang": "pt", + "layout": "normal", + "multiverse_ids": [ + 454775 + ], + "games": [ + "paper" + ] + }, + "expected": { + "artist": null, + "booster": false, + "border_color": null, + "card_back_id": "0aeebaf5-8c7d-4636-9e82-8c27447861f7", + "cmc": null, + "collector_number": "212", + "color_identity": [], + "colors": [], + "digital": false, + "finishes": [], + "full_art": false, + "games": [ + "paper" + ], + "highres_image": false, + "id": "01000000-0000-0000-0000-000000000003", + "illustration_id": null, + "image_status": null, + "image_uris": { + "art": "https://cards.scryfall.io/art/front/0/1/01000000-0000-0000-0000-000000000003.webp", + "art_crop": "https://cards.scryfall.io/art_crop/front/0/1/01000000-0000-0000-0000-000000000003.jpg", + "border_crop": "https://cards.scryfall.io/border_crop/front/0/1/01000000-0000-0000-0000-000000000003.jpg", + "crop": "https://cards.scryfall.io/crop/front/0/1/01000000-0000-0000-0000-000000000003.webp", + "display": "https://cards.scryfall.io/display/front/0/1/01000000-0000-0000-0000-000000000003.webp", + "grid": "https://cards.scryfall.io/grid/front/0/1/01000000-0000-0000-0000-000000000003.webp", + "large": "https://cards.scryfall.io/large/front/0/1/01000000-0000-0000-0000-000000000003.jpg", + "normal": "https://cards.scryfall.io/normal/front/0/1/01000000-0000-0000-0000-000000000003.jpg", + "png": "https://cards.scryfall.io/png/front/0/1/01000000-0000-0000-0000-000000000003.png", + "small": "https://cards.scryfall.io/small/front/0/1/01000000-0000-0000-0000-000000000003.jpg", + "thumb": "https://cards.scryfall.io/thumb/front/0/1/01000000-0000-0000-0000-000000000003.webp" + }, + "keywords": [], + "lang": "pt", + "layout": "normal", + "mana_cost": null, + "multiverse_ids": [ + 454775 + ], + "name": "Unmoored Ego", + "object": "card", + "oracle_id": "", + "oracle_text": null, + "oversized": false, + "prices": { + "eur": null, + "eur_foil": null, + "tix": null, + "usd": null, + "usd_etched": null, + "usd_foil": null + }, + "prints_search_uri": "https://api.example/v1/cards/search?order=released&q=oracleid%3A&unique=prints", + "promo": false, + "purchase_uris": { + "cardhoarder": "https://www.cardhoarder.com/cards?data%5Bsearch%5D=Unmoored+Ego", + "cardmarket": "https://www.cardmarket.com/en/Magic/Products/Search?searchString=Unmoored+Ego", + "tcgplayer": "https://www.tcgplayer.com/search/magic/product?productLineName=magic&q=Unmoored+Ego&view=grid" + }, + "rarity": null, + "related_uris": { + "edhrec": "https://edhrec.com/route/?cc=Unmoored+Ego", + "gatherer": "https://gatherer.wizards.com/Pages/Card/Details.aspx?multiverseid=454775&printed=true", + "tcgplayer_infinite_articles": "https://www.tcgplayer.com/search/articles?productLineName=magic&q=Unmoored+Ego", + "tcgplayer_infinite_decks": "https://www.tcgplayer.com/search/decks?productLineName=magic&q=Unmoored+Ego" + }, + "released_at": null, + "reprint": false, + "reserved": false, + "rulings_uri": "https://api.example/v1/cards/01000000-0000-0000-0000-000000000003/rulings", + "scryfall_set_uri": "https://scryfall.com/sets/grn?utm_source=api", + "scryfall_uri": "https://scryfall.com/card/grn/212/pt/unmoored-ego?utm_source=api", + "set": "grn", + "set_id": null, + "set_name": null, + "set_search_uri": "https://api.example/v1/cards/search?order=set&q=e%3Agrn&unique=prints", + "set_type": null, + "set_uri": null, + "story_spotlight": false, + "textless": false, + "type_line": null, + "uri": "https://api.example/v1/cards/01000000-0000-0000-0000-000000000003", + "variation": false + } + }, + { + "case": "a two-image layout keeps colors and pictures on its faces", + "row": { + "scryfall_id": "01000000-0000-0000-0000-000000000004", + "name": "Delver of Secrets // Insectile Aberration", + "layout": "transform", + "colors": [ + "U" + ], + "color_identity": [ + "U" + ], + "set_code": "isd", + "collector_number": "51", + "lang": "en", + "illustration_id": "22222222-3333-4444-5555-666666666666", + "games": [ + "paper" + ], + "card_faces": [ + { + "name": "Delver of Secrets", + "mana_cost": "{U}", + "oracle_text": "Front." + }, + { + "name": "Insectile Aberration", + "mana_cost": "", + "oracle_text": "Back.", + "power": "3" + } + ] + }, + "expected": { + "artist": null, + "booster": false, + "border_color": null, + "card_faces": [ + { + "image_uris": { + "art": "https://cards.scryfall.io/art/front/0/1/01000000-0000-0000-0000-000000000004.webp", + "art_crop": "https://cards.scryfall.io/art_crop/front/0/1/01000000-0000-0000-0000-000000000004.jpg", + "border_crop": "https://cards.scryfall.io/border_crop/front/0/1/01000000-0000-0000-0000-000000000004.jpg", + "crop": "https://cards.scryfall.io/crop/front/0/1/01000000-0000-0000-0000-000000000004.webp", + "display": "https://cards.scryfall.io/display/front/0/1/01000000-0000-0000-0000-000000000004.webp", + "grid": "https://cards.scryfall.io/grid/front/0/1/01000000-0000-0000-0000-000000000004.webp", + "large": "https://cards.scryfall.io/large/front/0/1/01000000-0000-0000-0000-000000000004.jpg", + "normal": "https://cards.scryfall.io/normal/front/0/1/01000000-0000-0000-0000-000000000004.jpg", + "png": "https://cards.scryfall.io/png/front/0/1/01000000-0000-0000-0000-000000000004.png", + "small": "https://cards.scryfall.io/small/front/0/1/01000000-0000-0000-0000-000000000004.jpg", + "thumb": "https://cards.scryfall.io/thumb/front/0/1/01000000-0000-0000-0000-000000000004.webp" + }, + "mana_cost": "{U}", + "name": "Delver of Secrets", + "object": "card_face", + "oracle_text": "Front." + }, + { + "image_uris": { + "art": "https://cards.scryfall.io/art/back/0/1/01000000-0000-0000-0000-000000000004.webp", + "art_crop": "https://cards.scryfall.io/art_crop/back/0/1/01000000-0000-0000-0000-000000000004.jpg", + "border_crop": "https://cards.scryfall.io/border_crop/back/0/1/01000000-0000-0000-0000-000000000004.jpg", + "crop": "https://cards.scryfall.io/crop/back/0/1/01000000-0000-0000-0000-000000000004.webp", + "display": "https://cards.scryfall.io/display/back/0/1/01000000-0000-0000-0000-000000000004.webp", + "grid": "https://cards.scryfall.io/grid/back/0/1/01000000-0000-0000-0000-000000000004.webp", + "large": "https://cards.scryfall.io/large/back/0/1/01000000-0000-0000-0000-000000000004.jpg", + "normal": "https://cards.scryfall.io/normal/back/0/1/01000000-0000-0000-0000-000000000004.jpg", + "png": "https://cards.scryfall.io/png/back/0/1/01000000-0000-0000-0000-000000000004.png", + "small": "https://cards.scryfall.io/small/back/0/1/01000000-0000-0000-0000-000000000004.jpg", + "thumb": "https://cards.scryfall.io/thumb/back/0/1/01000000-0000-0000-0000-000000000004.webp" + }, + "mana_cost": "", + "name": "Insectile Aberration", + "object": "card_face", + "oracle_text": "Back.", + "power": "3" + } + ], + "cmc": null, + "collector_number": "51", + "color_identity": [ + "U" + ], + "digital": false, + "finishes": [], + "full_art": false, + "games": [ + "paper" + ], + "highres_image": false, + "id": "01000000-0000-0000-0000-000000000004", + "image_status": null, + "keywords": [], + "lang": "en", + "layout": "transform", + "multiverse_ids": [], + "name": "Delver of Secrets // Insectile Aberration", + "object": "card", + "oracle_id": "", + "oversized": false, + "prices": { + "eur": null, + "eur_foil": null, + "tix": null, + "usd": null, + "usd_etched": null, + "usd_foil": null + }, + "prints_search_uri": "https://api.example/v1/cards/search?order=released&q=oracleid%3A&unique=prints", + "promo": false, + "purchase_uris": { + "cardhoarder": "https://www.cardhoarder.com/cards?data%5Bsearch%5D=Delver+of+Secrets", + "cardmarket": "https://www.cardmarket.com/en/Magic/Products/Search?searchString=Delver+of+Secrets", + "tcgplayer": "https://www.tcgplayer.com/search/magic/product?productLineName=magic&q=Delver+of+Secrets&view=grid" + }, + "rarity": null, + "related_uris": { + "edhrec": "https://edhrec.com/route/?cc=Delver+of+Secrets", + "tcgplayer_infinite_articles": "https://www.tcgplayer.com/search/articles?productLineName=magic&q=Delver+of+Secrets+%2F%2F+Insectile+Aberration", + "tcgplayer_infinite_decks": "https://www.tcgplayer.com/search/decks?productLineName=magic&q=Delver+of+Secrets+%2F%2F+Insectile+Aberration" + }, + "released_at": null, + "reprint": false, + "reserved": false, + "rulings_uri": "https://api.example/v1/cards/01000000-0000-0000-0000-000000000004/rulings", + "scryfall_set_uri": "https://scryfall.com/sets/isd?utm_source=api", + "scryfall_uri": "https://scryfall.com/card/isd/51/delver-of-secrets-insectile-aberration?utm_source=api", + "set": "isd", + "set_id": null, + "set_name": null, + "set_search_uri": "https://api.example/v1/cards/search?order=set&q=e%3Aisd&unique=prints", + "set_type": null, + "set_uri": null, + "story_spotlight": false, + "textless": false, + "type_line": null, + "uri": "https://api.example/v1/cards/01000000-0000-0000-0000-000000000004", + "variation": false + } + }, + { + "case": "a one-image multi-face card joins its cost at top level", + "row": { + "scryfall_id": "01000000-0000-0000-0000-000000000005", + "name": "Fire // Ice", + "layout": "split", + "set_code": "apc", + "collector_number": "128", + "lang": "en", + "games": [ + "paper" + ], + "card_faces": [ + { + "name": "Fire", + "mana_cost": "{1}{R}", + "oracle_text": "Two damage." + }, + { + "name": "Ice", + "mana_cost": "{1}{U}", + "oracle_text": "Tap target." + } + ] + }, + "expected": { + "artist": null, + "booster": false, + "border_color": null, + "card_back_id": "0aeebaf5-8c7d-4636-9e82-8c27447861f7", + "card_faces": [ + { + "mana_cost": "{1}{R}", + "name": "Fire", + "object": "card_face", + "oracle_text": "Two damage." + }, + { + "mana_cost": "{1}{U}", + "name": "Ice", + "object": "card_face", + "oracle_text": "Tap target." + } + ], + "cmc": null, + "collector_number": "128", + "color_identity": [], + "colors": [], + "digital": false, + "finishes": [], + "full_art": false, + "games": [ + "paper" + ], + "highres_image": false, + "id": "01000000-0000-0000-0000-000000000005", + "illustration_id": null, + "image_status": null, + "image_uris": { + "art": "https://cards.scryfall.io/art/front/0/1/01000000-0000-0000-0000-000000000005.webp", + "art_crop": "https://cards.scryfall.io/art_crop/front/0/1/01000000-0000-0000-0000-000000000005.jpg", + "border_crop": "https://cards.scryfall.io/border_crop/front/0/1/01000000-0000-0000-0000-000000000005.jpg", + "crop": "https://cards.scryfall.io/crop/front/0/1/01000000-0000-0000-0000-000000000005.webp", + "display": "https://cards.scryfall.io/display/front/0/1/01000000-0000-0000-0000-000000000005.webp", + "grid": "https://cards.scryfall.io/grid/front/0/1/01000000-0000-0000-0000-000000000005.webp", + "large": "https://cards.scryfall.io/large/front/0/1/01000000-0000-0000-0000-000000000005.jpg", + "normal": "https://cards.scryfall.io/normal/front/0/1/01000000-0000-0000-0000-000000000005.jpg", + "png": "https://cards.scryfall.io/png/front/0/1/01000000-0000-0000-0000-000000000005.png", + "small": "https://cards.scryfall.io/small/front/0/1/01000000-0000-0000-0000-000000000005.jpg", + "thumb": "https://cards.scryfall.io/thumb/front/0/1/01000000-0000-0000-0000-000000000005.webp" + }, + "keywords": [], + "lang": "en", + "layout": "split", + "mana_cost": "{1}{R} // {1}{U}", + "multiverse_ids": [], + "name": "Fire // Ice", + "object": "card", + "oracle_id": "", + "oversized": false, + "prices": { + "eur": null, + "eur_foil": null, + "tix": null, + "usd": null, + "usd_etched": null, + "usd_foil": null + }, + "prints_search_uri": "https://api.example/v1/cards/search?order=released&q=oracleid%3A&unique=prints", + "promo": false, + "purchase_uris": { + "cardhoarder": "https://www.cardhoarder.com/cards?data%5Bsearch%5D=Fire+%2F%2F+Ice", + "cardmarket": "https://www.cardmarket.com/en/Magic/Products/Search?searchString=Fire+%2F%2F+Ice", + "tcgplayer": "https://www.tcgplayer.com/search/magic/product?productLineName=magic&q=Fire+%2F%2F+Ice&view=grid" + }, + "rarity": null, + "related_uris": { + "edhrec": "https://edhrec.com/route/?cc=Fire+%2F%2F+Ice", + "tcgplayer_infinite_articles": "https://www.tcgplayer.com/search/articles?productLineName=magic&q=Fire+%2F%2F+Ice", + "tcgplayer_infinite_decks": "https://www.tcgplayer.com/search/decks?productLineName=magic&q=Fire+%2F%2F+Ice" + }, + "released_at": null, + "reprint": false, + "reserved": false, + "rulings_uri": "https://api.example/v1/cards/01000000-0000-0000-0000-000000000005/rulings", + "scryfall_set_uri": "https://scryfall.com/sets/apc?utm_source=api", + "scryfall_uri": "https://scryfall.com/card/apc/128/fire-ice?utm_source=api", + "set": "apc", + "set_id": null, + "set_name": null, + "set_search_uri": "https://api.example/v1/cards/search?order=set&q=e%3Aapc&unique=prints", + "set_type": null, + "set_uri": null, + "story_spotlight": false, + "textless": false, + "type_line": null, + "uri": "https://api.example/v1/cards/01000000-0000-0000-0000-000000000005", + "variation": false + } + }, + { + "case": "a flipped back face with no cost is skipped in the join", + "row": { + "scryfall_id": "01000000-0000-0000-0000-000000000006", + "name": "Erayo // Erayo's Essence", + "layout": "flip", + "set_code": "chk", + "collector_number": "2", + "lang": "en", + "games": [ + "paper" + ], + "card_faces": [ + { + "name": "Erayo", + "mana_cost": "{1}{U}", + "oracle_text": "Front." + }, + { + "name": "Erayo's Essence", + "mana_cost": "", + "oracle_text": "Back." + } + ] + }, + "expected": { + "artist": null, + "booster": false, + "border_color": null, + "card_back_id": "0aeebaf5-8c7d-4636-9e82-8c27447861f7", + "card_faces": [ + { + "mana_cost": "{1}{U}", + "name": "Erayo", + "object": "card_face", + "oracle_text": "Front." + }, + { + "mana_cost": "", + "name": "Erayo's Essence", + "object": "card_face", + "oracle_text": "Back." + } + ], + "cmc": null, + "collector_number": "2", + "color_identity": [], + "colors": [], + "digital": false, + "finishes": [], + "full_art": false, + "games": [ + "paper" + ], + "highres_image": false, + "id": "01000000-0000-0000-0000-000000000006", + "illustration_id": null, + "image_status": null, + "image_uris": { + "art": "https://cards.scryfall.io/art/front/0/1/01000000-0000-0000-0000-000000000006.webp", + "art_crop": "https://cards.scryfall.io/art_crop/front/0/1/01000000-0000-0000-0000-000000000006.jpg", + "border_crop": "https://cards.scryfall.io/border_crop/front/0/1/01000000-0000-0000-0000-000000000006.jpg", + "crop": "https://cards.scryfall.io/crop/front/0/1/01000000-0000-0000-0000-000000000006.webp", + "display": "https://cards.scryfall.io/display/front/0/1/01000000-0000-0000-0000-000000000006.webp", + "grid": "https://cards.scryfall.io/grid/front/0/1/01000000-0000-0000-0000-000000000006.webp", + "large": "https://cards.scryfall.io/large/front/0/1/01000000-0000-0000-0000-000000000006.jpg", + "normal": "https://cards.scryfall.io/normal/front/0/1/01000000-0000-0000-0000-000000000006.jpg", + "png": "https://cards.scryfall.io/png/front/0/1/01000000-0000-0000-0000-000000000006.png", + "small": "https://cards.scryfall.io/small/front/0/1/01000000-0000-0000-0000-000000000006.jpg", + "thumb": "https://cards.scryfall.io/thumb/front/0/1/01000000-0000-0000-0000-000000000006.webp" + }, + "keywords": [], + "lang": "en", + "layout": "flip", + "mana_cost": "{1}{U}", + "multiverse_ids": [], + "name": "Erayo // Erayo's Essence", + "object": "card", + "oracle_id": "", + "oversized": false, + "prices": { + "eur": null, + "eur_foil": null, + "tix": null, + "usd": null, + "usd_etched": null, + "usd_foil": null + }, + "prints_search_uri": "https://api.example/v1/cards/search?order=released&q=oracleid%3A&unique=prints", + "promo": false, + "purchase_uris": { + "cardhoarder": "https://www.cardhoarder.com/cards?data%5Bsearch%5D=Erayo", + "cardmarket": "https://www.cardmarket.com/en/Magic/Products/Search?searchString=Erayo", + "tcgplayer": "https://www.tcgplayer.com/search/magic/product?productLineName=magic&q=Erayo&view=grid" + }, + "rarity": null, + "related_uris": { + "edhrec": "https://edhrec.com/route/?cc=Erayo", + "tcgplayer_infinite_articles": "https://www.tcgplayer.com/search/articles?productLineName=magic&q=Erayo+%2F%2F+Erayo%27s+Essence", + "tcgplayer_infinite_decks": "https://www.tcgplayer.com/search/decks?productLineName=magic&q=Erayo+%2F%2F+Erayo%27s+Essence" + }, + "released_at": null, + "reprint": false, + "reserved": false, + "rulings_uri": "https://api.example/v1/cards/01000000-0000-0000-0000-000000000006/rulings", + "scryfall_set_uri": "https://scryfall.com/sets/chk?utm_source=api", + "scryfall_uri": "https://scryfall.com/card/chk/2/erayo-erayos-essence?utm_source=api", + "set": "chk", + "set_id": null, + "set_name": null, + "set_search_uri": "https://api.example/v1/cards/search?order=set&q=e%3Achk&unique=prints", + "set_type": null, + "set_uri": null, + "story_spotlight": false, + "textless": false, + "type_line": null, + "uri": "https://api.example/v1/cards/01000000-0000-0000-0000-000000000006", + "variation": false + } + }, + { + "case": "a reversible printing drops the three keys its faces carry", + "row": { + "scryfall_id": "01000000-0000-0000-0000-000000000007", + "oracle_id": "33333333-4444-5555-6666-777777777777", + "name": "Propaganda // Propaganda", + "layout": "reversible_card", + "cmc": 3, + "type_line": "Enchantment", + "set_code": "sld", + "collector_number": "500", + "lang": "en", + "games": [ + "paper" + ], + "card_faces": [ + { + "name": "Propaganda", + "mana_cost": "{2}{U}", + "oracle_text": "Front." + }, + { + "name": "Propaganda", + "mana_cost": "{2}{U}", + "oracle_text": "Back." + } + ] + }, + "expected": { + "artist": null, + "booster": false, + "border_color": null, + "card_faces": [ + { + "cmc": 3.0, + "image_uris": { + "art": "https://cards.scryfall.io/art/front/0/1/01000000-0000-0000-0000-000000000007.webp", + "art_crop": "https://cards.scryfall.io/art_crop/front/0/1/01000000-0000-0000-0000-000000000007.jpg", + "border_crop": "https://cards.scryfall.io/border_crop/front/0/1/01000000-0000-0000-0000-000000000007.jpg", + "crop": "https://cards.scryfall.io/crop/front/0/1/01000000-0000-0000-0000-000000000007.webp", + "display": "https://cards.scryfall.io/display/front/0/1/01000000-0000-0000-0000-000000000007.webp", + "grid": "https://cards.scryfall.io/grid/front/0/1/01000000-0000-0000-0000-000000000007.webp", + "large": "https://cards.scryfall.io/large/front/0/1/01000000-0000-0000-0000-000000000007.jpg", + "normal": "https://cards.scryfall.io/normal/front/0/1/01000000-0000-0000-0000-000000000007.jpg", + "png": "https://cards.scryfall.io/png/front/0/1/01000000-0000-0000-0000-000000000007.png", + "small": "https://cards.scryfall.io/small/front/0/1/01000000-0000-0000-0000-000000000007.jpg", + "thumb": "https://cards.scryfall.io/thumb/front/0/1/01000000-0000-0000-0000-000000000007.webp" + }, + "mana_cost": "{2}{U}", + "name": "Propaganda", + "object": "card_face", + "oracle_id": "33333333-4444-5555-6666-777777777777", + "oracle_text": "Front." + }, + { + "cmc": 3.0, + "image_uris": { + "art": "https://cards.scryfall.io/art/back/0/1/01000000-0000-0000-0000-000000000007.webp", + "art_crop": "https://cards.scryfall.io/art_crop/back/0/1/01000000-0000-0000-0000-000000000007.jpg", + "border_crop": "https://cards.scryfall.io/border_crop/back/0/1/01000000-0000-0000-0000-000000000007.jpg", + "crop": "https://cards.scryfall.io/crop/back/0/1/01000000-0000-0000-0000-000000000007.webp", + "display": "https://cards.scryfall.io/display/back/0/1/01000000-0000-0000-0000-000000000007.webp", + "grid": "https://cards.scryfall.io/grid/back/0/1/01000000-0000-0000-0000-000000000007.webp", + "large": "https://cards.scryfall.io/large/back/0/1/01000000-0000-0000-0000-000000000007.jpg", + "normal": "https://cards.scryfall.io/normal/back/0/1/01000000-0000-0000-0000-000000000007.jpg", + "png": "https://cards.scryfall.io/png/back/0/1/01000000-0000-0000-0000-000000000007.png", + "small": "https://cards.scryfall.io/small/back/0/1/01000000-0000-0000-0000-000000000007.jpg", + "thumb": "https://cards.scryfall.io/thumb/back/0/1/01000000-0000-0000-0000-000000000007.webp" + }, + "mana_cost": "{2}{U}", + "name": "Propaganda", + "object": "card_face", + "oracle_id": "33333333-4444-5555-6666-777777777777", + "oracle_text": "Back." + } + ], + "collector_number": "500", + "color_identity": [], + "digital": false, + "finishes": [], + "full_art": false, + "games": [ + "paper" + ], + "highres_image": false, + "id": "01000000-0000-0000-0000-000000000007", + "image_status": null, + "keywords": [], + "lang": "en", + "layout": "reversible_card", + "multiverse_ids": [], + "name": "Propaganda // Propaganda", + "object": "card", + "oversized": false, + "prices": { + "eur": null, + "eur_foil": null, + "tix": null, + "usd": null, + "usd_etched": null, + "usd_foil": null + }, + "prints_search_uri": "https://api.example/v1/cards/search?order=released&q=oracleid%3A33333333-4444-5555-6666-777777777777&unique=prints", + "promo": false, + "purchase_uris": { + "cardhoarder": "https://www.cardhoarder.com/cards?data%5Bsearch%5D=Propaganda+%2F%2F+Propaganda", + "cardmarket": "https://www.cardmarket.com/en/Magic/Products/Search?searchString=Propaganda+%2F%2F+Propaganda", + "tcgplayer": "https://www.tcgplayer.com/search/magic/product?productLineName=magic&q=Propaganda+%2F%2F+Propaganda&view=grid" + }, + "rarity": null, + "related_uris": { + "edhrec": "https://edhrec.com/route/?cc=Propaganda+%2F%2F+Propaganda", + "tcgplayer_infinite_articles": "https://www.tcgplayer.com/search/articles?productLineName=magic&q=Propaganda+%2F%2F+Propaganda", + "tcgplayer_infinite_decks": "https://www.tcgplayer.com/search/decks?productLineName=magic&q=Propaganda+%2F%2F+Propaganda" + }, + "released_at": null, + "reprint": false, + "reserved": false, + "rulings_uri": "https://api.example/v1/cards/01000000-0000-0000-0000-000000000007/rulings", + "scryfall_set_uri": "https://scryfall.com/sets/sld?utm_source=api", + "scryfall_uri": "https://scryfall.com/card/sld/500/propaganda-propaganda?utm_source=api", + "set": "sld", + "set_id": null, + "set_name": null, + "set_search_uri": "https://api.example/v1/cards/search?order=set&q=e%3Asld&unique=prints", + "set_type": null, + "set_uri": null, + "story_spotlight": false, + "textless": false, + "uri": "https://api.example/v1/cards/01000000-0000-0000-0000-000000000007", + "variation": false + } + }, + { + "case": "no marketplace ids means a name search per key", + "row": { + "scryfall_id": "01000000-0000-0000-0000-000000000008", + "name": "Invasion of Alara // Awaken the Maelstrom", + "layout": "modal_dfc", + "set_code": "mom", + "collector_number": "195", + "lang": "en", + "games": [ + "paper" + ], + "card_faces": [ + { + "name": "Invasion of Alara", + "mana_cost": "{W}{U}{B}{R}{G}", + "oracle_text": "Front." + }, + { + "name": "Awaken the Maelstrom", + "mana_cost": "", + "oracle_text": "Back." + } + ] + }, + "expected": { + "artist": null, + "booster": false, + "border_color": null, + "card_faces": [ + { + "image_uris": { + "art": "https://cards.scryfall.io/art/front/0/1/01000000-0000-0000-0000-000000000008.webp", + "art_crop": "https://cards.scryfall.io/art_crop/front/0/1/01000000-0000-0000-0000-000000000008.jpg", + "border_crop": "https://cards.scryfall.io/border_crop/front/0/1/01000000-0000-0000-0000-000000000008.jpg", + "crop": "https://cards.scryfall.io/crop/front/0/1/01000000-0000-0000-0000-000000000008.webp", + "display": "https://cards.scryfall.io/display/front/0/1/01000000-0000-0000-0000-000000000008.webp", + "grid": "https://cards.scryfall.io/grid/front/0/1/01000000-0000-0000-0000-000000000008.webp", + "large": "https://cards.scryfall.io/large/front/0/1/01000000-0000-0000-0000-000000000008.jpg", + "normal": "https://cards.scryfall.io/normal/front/0/1/01000000-0000-0000-0000-000000000008.jpg", + "png": "https://cards.scryfall.io/png/front/0/1/01000000-0000-0000-0000-000000000008.png", + "small": "https://cards.scryfall.io/small/front/0/1/01000000-0000-0000-0000-000000000008.jpg", + "thumb": "https://cards.scryfall.io/thumb/front/0/1/01000000-0000-0000-0000-000000000008.webp" + }, + "mana_cost": "{W}{U}{B}{R}{G}", + "name": "Invasion of Alara", + "object": "card_face", + "oracle_text": "Front." + }, + { + "image_uris": { + "art": "https://cards.scryfall.io/art/back/0/1/01000000-0000-0000-0000-000000000008.webp", + "art_crop": "https://cards.scryfall.io/art_crop/back/0/1/01000000-0000-0000-0000-000000000008.jpg", + "border_crop": "https://cards.scryfall.io/border_crop/back/0/1/01000000-0000-0000-0000-000000000008.jpg", + "crop": "https://cards.scryfall.io/crop/back/0/1/01000000-0000-0000-0000-000000000008.webp", + "display": "https://cards.scryfall.io/display/back/0/1/01000000-0000-0000-0000-000000000008.webp", + "grid": "https://cards.scryfall.io/grid/back/0/1/01000000-0000-0000-0000-000000000008.webp", + "large": "https://cards.scryfall.io/large/back/0/1/01000000-0000-0000-0000-000000000008.jpg", + "normal": "https://cards.scryfall.io/normal/back/0/1/01000000-0000-0000-0000-000000000008.jpg", + "png": "https://cards.scryfall.io/png/back/0/1/01000000-0000-0000-0000-000000000008.png", + "small": "https://cards.scryfall.io/small/back/0/1/01000000-0000-0000-0000-000000000008.jpg", + "thumb": "https://cards.scryfall.io/thumb/back/0/1/01000000-0000-0000-0000-000000000008.webp" + }, + "mana_cost": "", + "name": "Awaken the Maelstrom", + "object": "card_face", + "oracle_text": "Back." + } + ], + "cmc": null, + "collector_number": "195", + "color_identity": [], + "digital": false, + "finishes": [], + "full_art": false, + "games": [ + "paper" + ], + "highres_image": false, + "id": "01000000-0000-0000-0000-000000000008", + "image_status": null, + "keywords": [], + "lang": "en", + "layout": "modal_dfc", + "multiverse_ids": [], + "name": "Invasion of Alara // Awaken the Maelstrom", + "object": "card", + "oracle_id": "", + "oversized": false, + "prices": { + "eur": null, + "eur_foil": null, + "tix": null, + "usd": null, + "usd_etched": null, + "usd_foil": null + }, + "prints_search_uri": "https://api.example/v1/cards/search?order=released&q=oracleid%3A&unique=prints", + "promo": false, + "purchase_uris": { + "cardhoarder": "https://www.cardhoarder.com/cards?data%5Bsearch%5D=Invasion+of+Alara", + "cardmarket": "https://www.cardmarket.com/en/Magic/Products/Search?searchString=Invasion+of+Alara", + "tcgplayer": "https://www.tcgplayer.com/search/magic/product?productLineName=magic&q=Invasion+of+Alara&view=grid" + }, + "rarity": null, + "related_uris": { + "edhrec": "https://edhrec.com/route/?cc=Invasion+of+Alara", + "tcgplayer_infinite_articles": "https://www.tcgplayer.com/search/articles?productLineName=magic&q=Invasion+of+Alara+%2F%2F+Awaken+the+Maelstrom", + "tcgplayer_infinite_decks": "https://www.tcgplayer.com/search/decks?productLineName=magic&q=Invasion+of+Alara+%2F%2F+Awaken+the+Maelstrom" + }, + "released_at": null, + "reprint": false, + "reserved": false, + "rulings_uri": "https://api.example/v1/cards/01000000-0000-0000-0000-000000000008/rulings", + "scryfall_set_uri": "https://scryfall.com/sets/mom?utm_source=api", + "scryfall_uri": "https://scryfall.com/card/mom/195/invasion-of-alara-awaken-the-maelstrom?utm_source=api", + "set": "mom", + "set_id": null, + "set_name": null, + "set_search_uri": "https://api.example/v1/cards/search?order=set&q=e%3Amom&unique=prints", + "set_type": null, + "set_uri": null, + "story_spotlight": false, + "textless": false, + "type_line": null, + "uri": "https://api.example/v1/cards/01000000-0000-0000-0000-000000000008", + "variation": false + } + }, + { + "case": "an arena-only printing omits purchase_uris entirely", + "row": { + "scryfall_id": "01000000-0000-0000-0000-000000000009", + "name": "Alrund's Epiphany", + "set_code": "khm", + "collector_number": "A-198", + "lang": "en", + "layout": "normal", + "digital": true, + "games": [ + "arena" + ] + }, + "expected": { + "artist": null, + "booster": false, + "border_color": null, + "card_back_id": "0aeebaf5-8c7d-4636-9e82-8c27447861f7", + "cmc": null, + "collector_number": "A-198", + "color_identity": [], + "colors": [], + "digital": true, + "finishes": [], + "full_art": false, + "games": [ + "arena" + ], + "highres_image": false, + "id": "01000000-0000-0000-0000-000000000009", + "illustration_id": null, + "image_status": null, + "image_uris": { + "art": "https://cards.scryfall.io/art/front/0/1/01000000-0000-0000-0000-000000000009.webp", + "art_crop": "https://cards.scryfall.io/art_crop/front/0/1/01000000-0000-0000-0000-000000000009.jpg", + "border_crop": "https://cards.scryfall.io/border_crop/front/0/1/01000000-0000-0000-0000-000000000009.jpg", + "crop": "https://cards.scryfall.io/crop/front/0/1/01000000-0000-0000-0000-000000000009.webp", + "display": "https://cards.scryfall.io/display/front/0/1/01000000-0000-0000-0000-000000000009.webp", + "grid": "https://cards.scryfall.io/grid/front/0/1/01000000-0000-0000-0000-000000000009.webp", + "large": "https://cards.scryfall.io/large/front/0/1/01000000-0000-0000-0000-000000000009.jpg", + "normal": "https://cards.scryfall.io/normal/front/0/1/01000000-0000-0000-0000-000000000009.jpg", + "png": "https://cards.scryfall.io/png/front/0/1/01000000-0000-0000-0000-000000000009.png", + "small": "https://cards.scryfall.io/small/front/0/1/01000000-0000-0000-0000-000000000009.jpg", + "thumb": "https://cards.scryfall.io/thumb/front/0/1/01000000-0000-0000-0000-000000000009.webp" + }, + "keywords": [], + "lang": "en", + "layout": "normal", + "mana_cost": null, + "multiverse_ids": [], + "name": "Alrund's Epiphany", + "object": "card", + "oracle_id": "", + "oracle_text": null, + "oversized": false, + "prices": { + "eur": null, + "eur_foil": null, + "tix": null, + "usd": null, + "usd_etched": null, + "usd_foil": null + }, + "prints_search_uri": "https://api.example/v1/cards/search?order=released&q=oracleid%3A&unique=prints", + "promo": false, + "rarity": null, + "related_uris": { + "edhrec": "https://edhrec.com/route/?cc=Alrund%27s+Epiphany", + "tcgplayer_infinite_articles": "https://www.tcgplayer.com/search/articles?productLineName=magic&q=Alrund%27s+Epiphany", + "tcgplayer_infinite_decks": "https://www.tcgplayer.com/search/decks?productLineName=magic&q=Alrund%27s+Epiphany" + }, + "released_at": null, + "reprint": false, + "reserved": false, + "rulings_uri": "https://api.example/v1/cards/01000000-0000-0000-0000-000000000009/rulings", + "scryfall_set_uri": "https://scryfall.com/sets/khm?utm_source=api", + "scryfall_uri": "https://scryfall.com/card/khm/A-198/alrunds-epiphany?utm_source=api", + "set": "khm", + "set_id": null, + "set_name": null, + "set_search_uri": "https://api.example/v1/cards/search?order=set&q=e%3Akhm&unique=prints", + "set_type": null, + "set_uri": null, + "story_spotlight": false, + "textless": false, + "type_line": null, + "uri": "https://api.example/v1/cards/01000000-0000-0000-0000-000000000009", + "variation": false + } + }, + { + "case": "an empty string is a value, not an absence", + "row": { + "scryfall_id": "01000000-0000-0000-0000-00000000000a", + "name": "Island", + "mana_cost": "", + "cmc": 0, + "type_line": "Basic Land — Island", + "oracle_text": "", + "artist": "", + "set_code": "lea", + "collector_number": "288", + "lang": "en", + "layout": "normal", + "games": [ + "paper" + ] + }, + "expected": { + "artist": "", + "booster": false, + "border_color": null, + "card_back_id": "0aeebaf5-8c7d-4636-9e82-8c27447861f7", + "cmc": 0.0, + "collector_number": "288", + "color_identity": [], + "colors": [], + "digital": false, + "finishes": [], + "full_art": false, + "games": [ + "paper" + ], + "highres_image": false, + "id": "01000000-0000-0000-0000-00000000000a", + "illustration_id": null, + "image_status": null, + "image_uris": { + "art": "https://cards.scryfall.io/art/front/0/1/01000000-0000-0000-0000-00000000000a.webp", + "art_crop": "https://cards.scryfall.io/art_crop/front/0/1/01000000-0000-0000-0000-00000000000a.jpg", + "border_crop": "https://cards.scryfall.io/border_crop/front/0/1/01000000-0000-0000-0000-00000000000a.jpg", + "crop": "https://cards.scryfall.io/crop/front/0/1/01000000-0000-0000-0000-00000000000a.webp", + "display": "https://cards.scryfall.io/display/front/0/1/01000000-0000-0000-0000-00000000000a.webp", + "grid": "https://cards.scryfall.io/grid/front/0/1/01000000-0000-0000-0000-00000000000a.webp", + "large": "https://cards.scryfall.io/large/front/0/1/01000000-0000-0000-0000-00000000000a.jpg", + "normal": "https://cards.scryfall.io/normal/front/0/1/01000000-0000-0000-0000-00000000000a.jpg", + "png": "https://cards.scryfall.io/png/front/0/1/01000000-0000-0000-0000-00000000000a.png", + "small": "https://cards.scryfall.io/small/front/0/1/01000000-0000-0000-0000-00000000000a.jpg", + "thumb": "https://cards.scryfall.io/thumb/front/0/1/01000000-0000-0000-0000-00000000000a.webp" + }, + "keywords": [], + "lang": "en", + "layout": "normal", + "mana_cost": "", + "multiverse_ids": [], + "name": "Island", + "object": "card", + "oracle_id": "", + "oracle_text": "", + "oversized": false, + "prices": { + "eur": null, + "eur_foil": null, + "tix": null, + "usd": null, + "usd_etched": null, + "usd_foil": null + }, + "prints_search_uri": "https://api.example/v1/cards/search?order=released&q=oracleid%3A&unique=prints", + "promo": false, + "purchase_uris": { + "cardhoarder": "https://www.cardhoarder.com/cards?data%5Bsearch%5D=Island", + "cardmarket": "https://www.cardmarket.com/en/Magic/Products/Search?searchString=Island", + "tcgplayer": "https://www.tcgplayer.com/search/magic/product?productLineName=magic&q=Island&view=grid" + }, + "rarity": null, + "related_uris": { + "edhrec": "https://edhrec.com/route/?cc=Island", + "tcgplayer_infinite_articles": "https://www.tcgplayer.com/search/articles?productLineName=magic&q=Island", + "tcgplayer_infinite_decks": "https://www.tcgplayer.com/search/decks?productLineName=magic&q=Island" + }, + "released_at": null, + "reprint": false, + "reserved": false, + "rulings_uri": "https://api.example/v1/cards/01000000-0000-0000-0000-00000000000a/rulings", + "scryfall_set_uri": "https://scryfall.com/sets/lea?utm_source=api", + "scryfall_uri": "https://scryfall.com/card/lea/288/island?utm_source=api", + "set": "lea", + "set_id": null, + "set_name": null, + "set_search_uri": "https://api.example/v1/cards/search?order=set&q=e%3Alea&unique=prints", + "set_type": null, + "set_uri": null, + "story_spotlight": false, + "textless": false, + "type_line": "Basic Land — Island", + "uri": "https://api.example/v1/cards/01000000-0000-0000-0000-00000000000a", + "variation": false + } + } + ] +} diff --git a/api/scryfall_compat/mana.py b/api/scryfall_compat/mana.py new file mode 100644 index 000000000..441f2e7ce --- /dev/null +++ b/api/scryfall_compat/mana.py @@ -0,0 +1,420 @@ +"""Mana cost parsing for `GET /symbology/parse-mana`. + +The one reference endpoint that is computed rather than mirrored: it takes a cost written any way a +human might write it (`RUW`, `2WW`, `{X}{R}{R}`) and returns Scryfall's normalized form plus the +colors, mana value and the three colour-count flags. + +Two behaviours here were measured against api.scryfall.com on 2026-08-11 rather than inferred, both +because nothing documents them: + +- **The normalized cost reorders colored pips into the canonical colour order**, so `RUW` comes back + as `{U}{R}{W}` (Jeskai) and not as it was written. `_canonical_colors` is that rule. +- **The emission order is X, then generic, then every other pip in `GET /symbology` catalog + order**, regardless of where they appeared in the input: `2XWU` normalizes to `{X}{2}{W}{U}`, and + `CW` to `{W}{C}`. Generic pips are summed into one symbol, so `1{1}` is `{2}`. `_PIP_ORDER` is + that rule. +""" + +from __future__ import annotations + +import re +from typing import Any, NamedTuple + +# The colour wheel. Every canonical ordering is a walk around this cycle. +_WUBRG = ("W", "U", "B", "R", "G") +_COLOR_INDEX = {color: index for index, color in enumerate(_WUBRG)} + +# Symbols that are mana but not a colour: colorless, snow, and the energy-style pips. +_COLORLESS_PIPS = frozenset({"C", "S"}) + +# Variable pips, which contribute nothing to mana value. +_VARIABLE_PIPS = frozenset({"X", "Y", "Z"}) + +_BRACED = re.compile(r"\{([^}]*)\}") + +# Half-mana symbols are written {HW}; the half applies to the symbol that follows the H. +# `GET /symbology` lists exactly these two (2026-08-28), and reading `H` as a prefix over any colour +# was one symbol too generous: `?cost={HB}` is a 422 on api.scryfall.com, measured the same day. +_HALF_SYMBOLS = frozenset({"HW", "HR"}) +_HALF_MANA = 0.5 + +# Every hybrid symbol api.scryfall.com knows, in the order `GET /symbology` lists them. +# +# Fetched whole on 2026-08-28: 84 symbols, 36 of them hybrids, and this is all 36. The inventory is +# the rule -- a hybrid parses if and only if it is one of these -- because no rule stated in terms of +# the halves gets the boundary right. This module used to require exactly TWO halves, which rejects +# the ten PHYREXIAN HYBRIDS below: `{W/U/P}` and its nine siblings are printed symbols, and four live +# cards carry one in their mana cost (`is:phyrexian is:hybrid`, 2026-08-28 -- Ajani, Sleeper Agent +# `{1}{G}{G/W/P}{W}`; Tamiyo, Compleated Sage `{2}{G}{G/U/P}{U}`; Nahiri, the Unforgiving +# `{1}{R}{R/W/P}{W}`; Lukka, Bound to Ruin `{2}{R}{R/G/P}{G}`). Loosening the count to "two or three" +# would be just as wrong in the other direction: `{W/U/B}` and `{3/W}` are still 422s. +# +# The order is load-bearing twice over -- it is also the order hybrids are EMITTED in, see _PIP_ORDER. +_HYBRID_SYMBOLS = ( + # Colour pairs. + "W/U", + "W/B", + "B/R", + "B/G", + "U/B", + "U/R", + "R/G", + "R/W", + "G/W", + "G/U", + # Phyrexian hybrids -- one of two colours, or 2 life. All ten colour pairs exist. + "B/G/P", + "B/R/P", + "G/U/P", + "G/W/P", + "R/G/P", + "R/W/P", + "U/B/P", + "U/R/P", + "W/B/P", + "W/U/P", + # Colorless hybrids. + "C/W", + "C/U", + "C/B", + "C/R", + "C/G", + # Twobrid. + "2/W", + "2/U", + "2/B", + "2/R", + "2/G", + # Phyrexian. + "W/P", + "U/P", + "B/P", + "R/P", + "G/P", + "C/P", +) + +# Every SPELLING that names one of those symbols, mapped to the spelling Scryfall answers with. +# +# A two-part hybrid may be written either way round and comes back canonical -- measured one request +# each on 2026-08-28, once per family: `{U/W}`->`{W/U}`, `{W/2}`->`{2/W}`, `{P/W}`->`{W/P}`, +# `{W/C}`->`{C/W}`. A three-part one may NOT: `{U/W/P}` and `{P/W/U}` are both 422s where `{W/U/P}` +# parses, so the ten Phyrexian hybrids are accepted only as spelled above. +_HYBRID_CANONICAL = { + spelling: symbol + for symbol in _HYBRID_SYMBOLS + for spelling in ((symbol, "/".join(reversed(symbol.split("/")))) if symbol.count("/") == 1 else (symbol,)) +} + +# The order pips are EMITTED in: `GET /symbology` catalog order, for everything a cost can carry +# besides generic and variable pips (fetched 2026-08-28). +# +# Measured, one request per row on 2026-08-28, each written both ways round to prove it is a sort and +# not the writing order:: +# +# ?cost={G}{G/W}{W} {G/W}{G}{W} a hybrid comes out ahead of a plain pip of its colour +# ?cost={W}{HW} {HW}{W} so does a half pip +# ?cost={R}{HR}{R/W} {R/W}{HR}{R} and a hybrid comes out ahead of a half pip +# ?cost={W}{C/P} {C/P}{W} a colourless hybrid sorts with the hybrids, not at the end +# ?cost={S}{C} {C}{S} the colorless pips have an order of their own +# +# Between two hybrids it is THIS list's order and not the colour order, which is the one thing a +# colour-rank sort cannot express: `{G/W}{W/U}` answers `{W/U}{G/W}` and `{G/U}{W/B}` answers +# `{W/B}{G/U}`, both of which put the later colour first. Same for half pips: `{HR}{HW}` answers +# `{HW}{HR}` though Boros orders R before W. +# +# The five PLAIN colour pips are the exception, and the only one: `RUW` answers `{U}{R}{W}`, so they +# come out in canonical colour order rather than catalog order. They occupy five consecutive catalog +# slots, so ranking them within that block says exactly that -- see `_PLAIN_PIP_AT`. +_PIP_ORDER = (*_HYBRID_SYMBOLS, "HW", "HR", "W", "U", "B", "R", "G", "C", "S") +_PIP_INDEX = {symbol: index for index, symbol in enumerate(_PIP_ORDER)} +_PLAIN_PIP_AT = _PIP_INDEX["W"] + +# How many CHARACTERS of the joined fragment list the error names before Scryfall cuts it. +# +# 51, and it is characters rather than bytes -- measured across nine lengths on 2026-08-16: 51 `a`s +# come back whole and 52 come back as 51, while 51 `é`s (102 bytes) also come back whole and 60 come +# back as 51 characters / 102 bytes. The cut applies to the WHOLE joined list rather than per +# fragment: ten separate `{QQQQQQQQ}` tokens come back as 51 characters of the concatenation. There +# is no ellipsis, unlike `/cards/collection`'s 30-character echo -- the string simply stops. 51 is an +# odd bound and nothing here explains it, which is why the measurement travels with the constant. +_FRAGMENT_ECHO_LIMIT = 51 + + +class ManaCostError(ValueError): + """A fragment of the cost could not be understood as mana.""" + + +class _UnparseableSymbolError(Exception): + """One token is not mana. Internal; never raised out of this module. + + The message Scryfall sends names **every** unparseable fragment of the cost at once ("The + string fragment(s) ..."), so the fragments have to be collected before any error can be worded. + Raising the finished ``ManaCostError`` from ``_symbol_value`` reported only the first, and + reported it re-braced: ``?cost=!!!`` came back as ``“{!}”`` where api.scryfall.com says ``“!!!”``. + """ + + +def _canonical_colors(colors: set[str]) -> list[str]: + """Order a colour set the way Magic writes it. + + Every canonical ordering — allied pairs, enemy pairs, shards, wedges, four-colour runs and + WUBRG itself — is a walk around the colour wheel taking a constant number of steps: one step for + anything contiguous (`{G}{W}` for Selesnya, `{W}{U}{B}` for Esper), two for the arrangements that + are not (`{R}{W}` for Boros, `{U}{R}{W}` for Jeskai). Trying step 1 before step 2, and starting + points in WUBRG order, picks the same arrangement Scryfall does for all 31 colour combinations. + + Args: + colors: The colours present, in any order. + + Returns: + The colours in canonical order. + """ + if not colors: + return [] + wanted = {_COLOR_INDEX[color] for color in colors} + for step in (1, 2): + for start in range(len(_WUBRG)): + walk = [(start + offset * step) % len(_WUBRG) for offset in range(len(wanted))] + if set(walk) == wanted: + return [_WUBRG[index] for index in walk] + # Unreachable for any subset of WUBRG, but a colour set that is not one must still come back + # deterministically rather than as None. + return [color for color in _WUBRG if color in colors] + + +def _symbol_value(symbol: str) -> float: + """The mana value one braced symbol contributes. + + Args: + symbol: The symbol's inside, without braces, uppercased. + + Returns: + Its contribution to the total. + + Raises: + ManaCostError: If the symbol is not mana at all. + """ + if symbol.isdigit(): + return float(symbol) + if symbol in _VARIABLE_PIPS: + return 0.0 + if symbol in _HALF_SYMBOLS: + return _HALF_MANA + if "/" in symbol: + # A hybrid parses if and only if it is one of the 36 symbols Scryfall lists -- see + # _HYBRID_SYMBOLS for why the inventory rather than a rule about the halves. + canonical = _HYBRID_CANONICAL.get(symbol) + if canonical is None: + raise _UnparseableSymbolError + # A hybrid is worth its most expensive part: {2/W} is 2, {W/U}, {W/U/P} and {C/P} are 1. + return max(float(part) if part.isdigit() else 1.0 for part in canonical.split("/")) + if symbol in _COLORLESS_PIPS or symbol in _COLOR_INDEX: + return 1.0 + raise _UnparseableSymbolError + + +def _symbol_colors(symbol: str) -> set[str]: + """The colours one braced symbol contributes. + + Args: + symbol: The symbol's inside, without braces, uppercased. + + Returns: + The colours it produces; empty for generic, variable and colorless pips. + """ + return {part for part in re.split(r"[/]", symbol.removeprefix("H")) if part in _COLOR_INDEX} + + +class _Token(NamedTuple): + """One symbol, plus how it was written -- which the error wording needs and the rules do not.""" + + symbol: str + """The symbol's inside, brace-stripped and uppercased. Every parsing rule reads this.""" + spelling: str + """How it was WRITTEN, uppercased: a braced token keeps its braces, a bare character does not.""" + braced: bool + """True for a braced token, which is what stops `!!` and `{!}{!}` merging into one fragment.""" + + +def _tokenize(raw: str) -> list[_Token]: + """Split a written cost into braced-symbol contents. + + Unbraced runs are read a character at a time, except for digits, which group so `11R` is + `{11}{R}` rather than `{1}{1}{R}`. + + Args: + raw: The cost as written. + + Returns: + One entry per symbol. + """ + tokens: list[_Token] = [] + position = 0 + upper = raw.upper() + while position < len(upper): + braced = _BRACED.match(upper, position) + if braced: + tokens.append(_Token(braced.group(1).strip(), braced.group(0), braced=True)) + position = braced.end() + continue + char = upper[position] + if char.isdigit(): + digits = re.match(r"\d+", upper[position:]).group(0) + tokens.append(_Token(digits, digits, braced=False)) + position += len(digits) + continue + if not char.isspace(): + tokens.append(_Token(char, char, braced=False)) + position += 1 + return tokens + + +def _reported_fragment(token: _Token) -> str: + """How Scryfall names a part of the cost it could not read. + + Measured one request per row against api.scryfall.com on 2026-08-16:: + + ?cost=!!! “!!!” three bare characters, reported as ONE run + ?cost=é “É” uppercased, and reported as itself rather than re-braced + ?cost={QQQ} “{QQQ}” a braced token keeps its braces + ?cost={} “{}” including the empty one + ?cost={W/U/B} “{//}” the RECOGNIZED halves are struck out and the residue reported + + The last row is the rule the others are a degenerate case of: what comes back is the fragment + with everything Scryfall could read removed. `{QQQ}` keeps all three Qs because none of them is + a symbol; `{W/U/B}` keeps only its punctuation. + + What counts as "could read" is exactly the ten ONE-CHARACTER mana symbols -- the five colours, + `{C}`, `{S}` and the three variables. `P`, `H` and digits are NOT struck, which this used to get + wrong by inferring the set from what the parser prices rather than measuring it. Five more rows, + one request each on 2026-08-28, all of them costs the inventory above now rejects:: + + ?cost={U/W/P} “{//P}” a Phyrexian hybrid spelled backwards -- the P survives + ?cost={2/W/P} “{2//P}” and so does the generic half + ?cost={3/W} “{3/}” there is no {3/W}; only {2/X} twobrids exist + ?cost={H/W} “{H/}” H survives too + ?cost={HB} “{H}” there is no {HB} either; only {HW} and {HR} + + Args: + token: The token that could not be parsed. + + Returns: + The fragment as Scryfall would name it. + """ + if not token.braced: + return token.spelling + residue = "".join( + char for char in token.symbol if char not in _COLOR_INDEX and char not in _COLORLESS_PIPS and char not in _VARIABLE_PIPS + ) + return f"{{{residue}}}" + + +def parse_mana_cost(raw: str) -> dict[str, Any]: + """Build Scryfall's ManaCost object for a written cost. + + Args: + raw: The cost as the client wrote it. + + Returns: + The ManaCost object. + + Raises: + ManaCostError: If a fragment is not mana. + """ + tokens = _tokenize(raw or "") + + generic = 0 + variables: list[str] = [] + # One list for every pip that is neither generic nor variable -- colored, colorless and hybrid + # alike -- because their emission order is one catalog order and not three buckets: `{W}{C/P}` + # answers `{C/P}{W}`, so a colourless hybrid comes out AHEAD of a coloured pip. + pips: list[str] = [] + color_set: set[str] = set() + total = 0.0 + + # EVERY unparseable fragment is collected before any error is raised, because Scryfall's message + # names them all at once -- CONCATENATED IN ORDER WITH NO SEPARATOR, and the readable symbols + # between them do not separate them either. Measured 2026-08-16, one request per row:: + # + # ?cost={Q}W{T} “{Q}{T}” the readable {W} between them leaves no trace + # ?cost=!W! “!!” same, for bare characters + # ?cost=!{Q}! “!{Q}!” braced and bare interleave in written order + # ?cost=a{Q}b “A{Q}” `b` is BLACK MANA and readable, so only two fragments + # + # An earlier pass joined the fragments with a space, which no measurement supported and which + # `{Q}W{T}` disproves. One accumulated string is now the whole mechanism -- with an empty + # separator there is nothing left for a per-fragment merge rule to do. + bad = "" + for token in tokens: + try: + total += _symbol_value(token.symbol) + except _UnparseableSymbolError: + bad += _reported_fragment(token) + continue + color_set |= _symbol_colors(token.symbol) + if token.symbol.isdigit(): + generic += int(token.symbol) + elif token.symbol in _VARIABLE_PIPS: + variables.append(token.symbol) + else: + # A hybrid is emitted in the spelling Scryfall answers with, not the one it was written in. + pips.append(_HYBRID_CANONICAL.get(token.symbol, token.symbol)) + + if bad: + msg = f"The string fragment(s) “{bad[:_FRAGMENT_ECHO_LIMIT]}” could not be understood as part of mana cost." + raise ManaCostError(msg) + + colors = _canonical_colors(color_set) + # An empty cost is null, but a cost that was written and happens to be free is `{0}`: Scryfall + # answers `cost=` with null and `cost=0` with "{0}", so the two cannot share a branch. + cost = _render_cost(variables, generic, pips, colors) if tokens else None + + return { + "object": "mana_cost", + "cost": cost, + "colors": [color for color in _WUBRG if color in color_set], + "cmc": total, + "colorless": not color_set, + "monocolored": len(color_set) == 1, + "multicolored": len(color_set) > 1, + } + + +def _render_cost( + variables: list[str], + generic: int, + pips: list[str], + colors: list[str], +) -> str | None: + """Assemble the normalized cost string. + + Args: + variables: X/Y/Z pips, in the order written. + generic: Summed generic mana. + pips: Every other symbol, in the order written. + colors: The canonical colour order the five plain colour pips come out in. + + Returns: + The normalized cost. A cost whose symbols all cancel to nothing renders as `{0}`. + """ + rank = {color: index for index, color in enumerate(colors)} + + def sort_key(symbol: str) -> int: + # Catalog order, with the five plain colour pips ranked inside their own block of the + # catalog so that they alone come out in canonical colour order. Every pip that reaches here + # is a symbol Scryfall lists, so the lookup always hits. + if symbol in rank: + return _PLAIN_PIP_AT + rank[symbol] + return _PIP_INDEX.get(symbol, len(_PIP_ORDER)) + + ordered = sorted(pips, key=sort_key) + # Variables come out in X, Y, Z order regardless of how they were written, and repeats group: + # `?cost=xyzzy` is `{X}{Y}{Y}{Z}{Z}` on api.scryfall.com (measured 2026-08-16) where writing + # order gives `{X}{Y}{Z}{Z}{Y}`. A plain sort does both at once -- the alphabet and the pip + # order coincide. + parts = [f"{{{symbol}}}" for symbol in sorted(variables)] + if generic: + parts.append(f"{{{generic}}}") + parts.extend(f"{{{symbol}}}" for symbol in ordered) + return "".join(parts) or "{0}" diff --git a/api/scryfall_compat/objects.py b/api/scryfall_compat/objects.py new file mode 100644 index 000000000..bfc949382 --- /dev/null +++ b/api/scryfall_compat/objects.py @@ -0,0 +1,798 @@ +"""Scryfall response objects: card reconstruction, envelopes, and the text rendering. + +Everything here is pure — dicts in, dicts out — so the payload shape can be tested without a +database or a request. `routes.py` owns the HTTP and SQL sides. + +The one subtle piece is `to_scryfall_card`. `cards.raw_card_blob` holds the card object Scryfall +sent, but not quite untouched: `preprocess_card` adds three internal keys to it and normalizes an +absent `flavor_text` to `""`. Both are exactly reversible, and reversing them is the whole of the +function. + +There is no column holding a pristine copy alongside it, and there deliberately isn't one: the blob +being answerable is a property of the importer, maintained there rather than worked around here. +The one case where the blob is *not* the card — a multi-face row written before the merged-row work +— is handled as a fallback rather than as a stored duplicate, because it is a fixed window that one +import closes. +""" + +from __future__ import annotations + +import datetime +import re +import string +import urllib.parse +import uuid +from typing import Any + +# Keys `preprocess_card` adds to the object it snapshots into raw_card_blob. Stripping them, and +# undoing the flavor_text normalization below, inverts the snapshot. A multi-face row carries only +# `card_name`; a single-face one carries all three. +_IMPORTER_ADDED_KEYS = ("card_name", "face_name", "face_idx") + +# The `version` vocabulary of the image format -- SIX names, not the eleven `image_uris` carries. +# +# The two lists used to be the same list, and are not any more: Scryfall's `image_uris` gained five +# webp sizes (see _IMAGE_EXTENSIONS) that `version=` does not accept. Measured against +# api.scryfall.com on 2026-08-16 -- `?format=image&version=thumb` redirects to the LARGE jpg, byte +# for byte the same fallback `version=bogus` gets, and the same for grid/display/art/crop. So these +# five are emitted as URLs and refused as parameters, and widening this tuple to match the other +# would silently change five 302 targets. +IMAGE_VERSIONS = ("small", "normal", "large", "png", "art_crop", "border_crop") +DEFAULT_IMAGE_VERSION = "large" + +# Scryfall pages every card list at 175, and clients page by following `next_page` rather than by +# computing offsets, so this has to match or a client's page count silently disagrees with ours. +PAGE_SIZE = 175 + +# Scryfall caps a collection POST at 75 identifiers and 422s past it. +MAX_COLLECTION_IDENTIFIERS = 75 + +# Scryfall caps an autocomplete catalog at 20 names. +MAX_AUTOCOMPLETE_VALUES = 20 + + +# Every field the engine must return for a card object to be assembled. Passed as `fields=` on each +# lookup, so the engine emits exactly this and nothing is fetched that is never read. +CARD_OBJECT_FIELDS = ( + "name", "scryfall_id", "oracle_id", "layout", "mana_cost", "cmc", "type_line", "oracle_text", + "power", "toughness", "loyalty", "colors", "color_identity", "card_keywords", "set_code", "set_name", + "collector_number", "rarity", "flavor_text", "artist", "illustration_id", "released_at", + "legalities", "edhrec_rank", "price_usd", "price_eur", "price_tix", "watermark", + "card_frame_data", "card_is_tags", "border_color", "frame", + "lang", "image_status", "set_type", "security_stamp", "set_id", "arena_id", "mtgo_id", + "mtgo_foil_id", "tcgplayer_id", "tcgplayer_etched_id", "cardmarket_id", "penny_rank", + "image_updated_at", "price_usd_foil", "price_usd_etched", "price_eur_foil", "multiverse_ids", + "promo_types", "frame_effects", "games", "finishes", "booster", "digital", "foil", "nonfoil", + "full_art", "highres_image", "oversized", "promo", "reprint", "story_spotlight", "textless", + "variation", "card_faces", "all_parts", +) # fmt: skip + +# Scryfall's card back, one image for every normal card. +CARD_BACK_ID = "0aeebaf5-8c7d-4636-9e82-8c27447861f7" + +# The file extension each `image_uris` size is served as, in Scryfall's own key order. +# +# ELEVEN, not the six this module shipped with. Scryfall added five webp sizes -- `thumb`, `grid`, +# `display`, `art`, `crop` -- and every card object it serves carries all eleven; a six-key +# `image_uris` differed from Scryfall on every card object emitted. +# +# Unconditional, and measured that way: across all 540,484 printings in the 2026-08-16 all_cards +# bulk, `image_uris` is either wholly ABSENT (8,444 cards, 7,641 faces -- the layouts whose picture +# lives on the other level) or carries exactly these eleven keys in exactly this order. No card, +# face, layout or `image_status` carries a partial set, so there is no per-key conditionality to +# round-trip the way `printed_*` has. +# +# Derived, not stored: the same scan confirms all eleven URLs are the same pure function of the id +# and the face on every one of the 548,604 objects that has them -- `art_crop` and `art` are +# different sizes of one path, not a stored pair. These five cost zero storage, which is why they +# are a table and not a column. +_IMAGE_EXTENSIONS = { + "small": "jpg", + "normal": "jpg", + "large": "jpg", + "png": "png", + "art_crop": "jpg", + "border_crop": "jpg", + "thumb": "webp", + "grid": "webp", + "display": "webp", + "art": "webp", + "crop": "webp", +} + +# magic.cards column -> the engine's name for the same value. The columns predate the engine, and +# `to_scryfall_card` reads engine names. +_SQL_COLUMN_ALIASES = { + "card_name": "name", + "card_set_code": "set_code", + "mana_cost_text": "mana_cost", + "card_legalities": "legalities", + "card_layout": "layout", + "card_watermark": "watermark", + "card_artist": "artist", + "card_border": "border_color", +} + +_RARITY_BY_INT = {0: "common", 1: "uncommon", 2: "rare", 3: "mythic", 4: "special", 5: "bonus"} + + +def sql_row_to_engine_row(row: dict[str, Any]) -> dict[str, Any]: + """Reshape a `magic.cards` row into the shape the engine emits. + + There is ONE card-object builder and both paths go through it; this is what lets the SQL + fallback use it. Two builders would be two chances to disagree with Scryfall, and a fallback is + where a different answer is least affordable — it runs when the engine is already in trouble. + + Building from `raw_card_blob` instead is not an option: for a multi-face row the blob is the + FRONT FACE, not the card, so it would silently degrade exactly the cards the merge exists to fix. + + Args: + row: A row selected with routes._CARD_COLUMNS. + + Returns: + The same values under the engine's field names, with the compat residue flattened. + """ + out: dict[str, Any] = {} + for column, value in row.items(): + if column in ("card_compat_blob", "raw_card_blob", "card_colors", "card_color_identity"): + continue + # psycopg binds dates and uuids as objects; JSON carries neither, and the engine path + # already emits strings. Normalizing here is what keeps ONE builder viable. + if isinstance(value, datetime.date): + normalized: Any = value.isoformat() + elif isinstance(value, uuid.UUID): + normalized = str(value) + else: + normalized = value + out[_SQL_COLUMN_ALIASES.get(column, column)] = normalized + + # jsonb objects store these as {key: true} sets; the engine emits lists. + for target, column in (("colors", "card_colors"), ("color_identity", "card_color_identity")): + value = row.get(column) + out[target] = sorted(value) if isinstance(value, dict) else [] + for key in ("card_keywords", "card_is_tags"): + if isinstance(row.get(key), dict): + out[key] = sorted(row[key]) + + if row.get("card_rarity_int") is not None: + out["rarity"] = _RARITY_BY_INT.get(row["card_rarity_int"]) + + # The residue is one column here and individual fields on an engine row. + out.update(row.get("card_compat_blob") or {}) + return out + + +def _image_uris(scryfall_id: str, updated_at: int | None, face: str = "front") -> dict[str, str]: + """Build the CDN URLs for one face. + + Scryfall's paths are a pure function of the card id: its first two hex digits become directory + levels, and `image_updated_at` rides as a cache-buster. Nothing about these is stored. + """ + scryfall_id = str(scryfall_id or "") + if not scryfall_id: + return {} + suffix = f"?{updated_at}" if updated_at else "" + first, second = scryfall_id[0], scryfall_id[1] + return { + size: f"https://cards.scryfall.io/{size}/{face}/{first}/{second}/{scryfall_id}.{ext}{suffix}" + for size, ext in _IMAGE_EXTENSIONS.items() + } + + +# Characters Scryfall DELETES from a slug rather than hyphenating. Live-derived: "Erayo's Essence" +# slugs to `erayos-essence` (not `erayo-s-essence`), "S.H.I.E.L.D." to `shield`, `Henzie "Toolbox" +# Torre` to `henzie-toolbox-torre`, and the zhs printings of Kongming/Pang Tong pin the curly +# quotes. U+201E ("bottom quote") is NOT deleted -- `Henzie ,,Der Beschaffer" Torre` (de) keeps it. +_SLUG_DELETED = frozenset("'\",./\u201c\u201d") + +# Slug bytes served literally; every other byte is UTF-8 percent-encoded, uppercase hex. The literal +# set is exactly what appears un-encoded across the bulk corpus; `?` is the one ASCII special +# observed encoded. Unobserved characters encode, which can never break a URL. +_SLUG_LITERAL = frozenset(string.ascii_letters + string.digits + "!&()+-:;=_") + +# The languages Scryfall writes into the scryfall_uri path -- its ten print localizations, exactly. +# The glyph and novelty languages (ph, qya, he, la, grc, ar, sa, dw) get NO path segment: a ph Elesh +# Norn lives at `/card/one/414/elesh-norn-mother-of-machines`, English form. +_SLUG_LANG_SEGMENTS = frozenset({"de", "es", "fr", "it", "ja", "ko", "pt", "ru", "zhs", "zht"}) + + +def _slug(name: str) -> str: + """Scryfall's URL slug for a card name. + + NOT the folklore "non-alphanumerics collapse to hyphens" rule this used to carry -- that + hyphenates apostrophes (`erayo-s-essence`) and serves raw UTF-8 (`jötun-grunt`) where production + Scryfall deletes the apostrophe and percent-encodes the bytes. The real rule, verified against + the `scryfall_uri` of all 540,484 printings in the 2026-08-16 all_cards bulk (zero mismatches): + + 1. lowercase; + 2. DELETE `' " , . /` and the curly quotes U+201C/U+201D; + 3. each run of ASCII spaces becomes one hyphen -- literal hyphens pass through and may stack + (ru "Пламенник - военный разведчик" keeps `---`), and nothing is trimmed ("Humming-" and + "With Great Power . . ." both keep their trailing hyphen); + 4. everything else survives verbatim (`:`, `!`, `&`, and CJK punctuation) and is then UTF-8 + percent-encoded per _SLUG_LITERAL. + """ + cleaned = "".join(c for c in name.lower() if c not in _SLUG_DELETED) + hyphenated = re.sub(" +", "-", cleaned) + return "".join( + chr(b) if chr(b) in _SLUG_LITERAL else f"%{b:02X}" for b in hyphenated.encode("utf-8") + ) + + +def _scryfall_uri(name: str, set_code: str, number: str, lang: str) -> str: + """`https://scryfall.com/card/{set}/{number}[/{lang}]/{slug}?utm_source=api`. + + A foreign printing keeps the language segment and takes the plain English slug (ody/243/zhs -> + `/zhs/holistic-wisdom`, verified live). Scryfall also writes a + `slug(printed name)-(slug(english name))` path where it HAS a printed name (grn/212/pt is + `ego-%C3%A0-deriva-(unmoored-ego)`); reproducing that needs the printed name, which the card + row does not carry yet, and the English fallback is what it serves until then. + """ + segment = f"{lang}/" if lang in _SLUG_LANG_SEGMENTS else "" + return f"https://scryfall.com/card/{set_code}/{number}/{segment}{_slug(name)}?utm_source=api" + + +# The layouts Scryfall gives TWO images to -- the ones that are two pieces of cardboard, or a +# front and a back. Everything else with `card_faces` (split, flip, adventure, prepare) is ONE +# image, and its faces must NOT get per-face image_uris: doing so invents a `.../back/...` URL with +# no image behind it. Verified exhaustively against the 2026-08-16 all_cards bulk, zero exceptions +# in either direction. These layouts also keep `colors`, `card_back_id` and `illustration_id` on +# their faces alone. +_TWO_IMAGE_LAYOUTS = frozenset({"art_series", "double_faced_token", "modal_dfc", "reversible_card", "transform"}) + +# A REVERSIBLE printing keeps NOTHING of the card at top level -- not even the three keys every +# other multi-face layout keeps. Measured across the whole 2026-08-16 all_cards bulk: all 81 omit +# `oracle_id`, `cmc` and `type_line`, where a `transform` printing sends all three. Its FACES carry +# the card's `oracle_id` and `cmc` instead, 0 of 81 disagreeing, so omitting them loses nothing. +_REVERSIBLE_LAYOUT = "reversible_card" + +# The layouts a SEARCH LINK spells with the JOINED name -- `related_uris.edhrec` and all three +# marketplace fallbacks in `purchase_uris`, which take one and the same string. Every other +# multi-face layout searches the FRONT face -- verified card for card against api.scryfall.com. +# +# THE MARKETPLACES SPLIT THE SAME WAY, which is why this is no longer edhrec's list alone. Measured +# on api.scryfall.com 2026-08-31 over `unique=prints`, on the first printing of each card whose ids +# are MISSING so the SEARCH form is what gets emitted, reading the tcgplayer term out of the `u=` +# parameter of Scryfall's own partner redirect: +# +# split Bind // Liberate cmb2/88 cardhoarder `Bind // Liberate` +# reversible_card Mechtitan // Mechtitan sld/1969 cardhoarder `Mechtitan // Mechtitan` +# double_faced_token Snake // Zombie cc2/9 all three, `Snake // Zombie` +# split Who // What // When // Where // Why und/75 cardhoarder the whole name +# adventure Champions of Archery // Join the … ph19/4 `Champions of Archery` +# flip Curse of the Fire Penguin // … unh/73 `Curse of the Fire Penguin` +# art_series Aang and Katara // Aang and Katara atle/8 `Aang and Katara` +# transform Delver of Secrets // Insectile … sld/2367 `Delver of Secrets` +# +# The two `tcgplayer_infinite_*` searches in `related_uris` are the exception that stays: they take +# the joined name on EVERY layout, so those two and this are deliberately not one string. +_JOINED_SEARCH_LAYOUTS = frozenset({"double_faced_token", "reversible_card", "split"}) + + +def _related_uris(name: str, search_name: str, multiverse_ids: list[Any], lang: str) -> dict[str, str]: + """Scryfall's `related_uris`, pointing at the destinations directly. + + Scryfall wraps the TCGplayer entries in `partner.tcgplayer.com/...?u=` with + its own affiliate code. The destination is the same page, and emitting the wrapper from this + host would route another service's affiliate revenue to Scryfall. + + `gatherer` LEADS the object when the printing has multiverse ids, built from the FIRST id, with + `printed=true` for every non-English printing and `printed=false` for English -- verified + against the bulk corpus at 540,430 of 540,484 printings. The 54 exceptions are foreign-only + promos (dd2-ja, snc launch, one-ph, ltc-qya) whose Gatherer entries carry no translation; that + fact lives on Scryfall's side of the wire and is not derivable from the row. + """ + out: dict[str, str] = {} + first_id = multiverse_ids[0] if multiverse_ids else None + if isinstance(first_id, int): + printed = "false" if lang == "en" else "true" + out["gatherer"] = ( + f"https://gatherer.wizards.com/Pages/Card/Details.aspx?multiverseid={first_id}&printed={printed}" + ) + quoted = urllib.parse.quote_plus(name) + out["tcgplayer_infinite_articles"] = ( + f"https://www.tcgplayer.com/search/articles?productLineName=magic&q={quoted}" + ) + out["tcgplayer_infinite_decks"] = f"https://www.tcgplayer.com/search/decks?productLineName=magic&q={quoted}" + out["edhrec"] = f"https://edhrec.com/route/?cc={urllib.parse.quote_plus(search_name)}" + return out + + +def _purchase_uris(row: dict[str, Any], search_name: str) -> dict[str, str]: + """Scryfall's `purchase_uris`, product links where the ids exist and name searches where not. + + Rebuilt from the marketplace ids -- or, for a key whose id this printing does not have, from a + NAME SEARCH on that marketplace. Same affiliate reasoning as `_related_uris`. + + All three keys are always present. The fallback is per KEY, not per card: an English printing + with TCGplayer and Cardmarket ids but no MTGO id gets two product links and a cardhoarder + search (verified live across khm). Every foreign printing takes the search form on all three -- + marketplace product ids belong to the English printing. Emitting nothing was the alternative, + and it made `purchase_uris` an empty object on 426,416 printings. + + `search_name` IS `_related_uris`' -- the caller decides the string, and all three marketplaces + split by layout exactly the way edhrec does (the measurements are on _JOINED_SEARCH_LAYOUTS). + This took the joined name and cut the front face off it here, on EVERY layout, which searched + for `Snake // Zombie` (cc2/9) as `Snake` and `Who // What // When // Where // Why` (und/75) as + `Who` against a Scryfall that spells both whole. On a transforming card the front face is still + right -- `Invasion of Alara`, not `Invasion of Alara // Awaken the Maelstrom` -- because there + the joined string matches no product. + """ + q = urllib.parse.quote_plus(search_name) + tcg, cm, mtgo = row.get("tcgplayer_id"), row.get("cardmarket_id"), row.get("mtgo_id") + return { + "tcgplayer": ( + f"https://www.tcgplayer.com/product/{tcg}?page=1" + if tcg + else f"https://www.tcgplayer.com/search/magic/product?productLineName=magic&q={q}&view=grid" + ), + "cardmarket": ( + f"https://www.cardmarket.com/en/Magic/Products?idProduct={cm}" + if cm + else f"https://www.cardmarket.com/en/Magic/Products/Search?searchString={q}" + ), + "cardhoarder": ( + f"https://www.cardhoarder.com/cards/{mtgo}" + if mtgo + else f"https://www.cardhoarder.com/cards?data%5Bsearch%5D={q}" + ), + } + + +def _prices(row: dict[str, Any]) -> dict[str, Any]: + """Scryfall's `prices` object: the three price columns plus the three residue variants.""" + + def fmt(value: float | None) -> str | None: + return None if value is None else f"{float(value):.2f}" + + return { + "usd": fmt(row.get("price_usd")), + "usd_foil": fmt(row.get("price_usd_foil")), + "usd_etched": fmt(row.get("price_usd_etched")), + "eur": fmt(row.get("price_eur")), + "eur_foil": fmt(row.get("price_eur_foil")), + "tix": fmt(row.get("price_tix")), + } + + +def _faces(row: dict[str, Any], *, two_image: bool, reversible: bool) -> list[dict[str, Any]]: + """The card's faces, with the keys the engine deliberately does not store re-added. + + `object` is the constant "card_face", and a face's `image_uris` is the card's CDN function with + front/back swapped, so neither is worth archive space. + + `image_uris` is gated on the LAYOUT rather than on the face count: only a two-image layout has + a second picture, and giving one to a split or adventure face invents a URL with nothing behind + it. An empty `mana_cost` or `oracle_text` on a face is a VALUE, not an omission -- every face + of every multi-face printing in the corpus carries both keys (8,620 of 8,620 transform faces, + 4,356 of them with an empty cost), so an empty string there is a costless back face. + """ + faces = row.get("card_faces") or [] + out = [] + for index, face in enumerate(faces): + built: dict[str, Any] = {"object": "card_face"} + built.update( + { + key: value + for key, value in face.items() + if value is not None and (value not in ("", []) or key in ("mana_cost", "oracle_text")) + } + ) + if reversible: + # Both faces of a reversible printing carry the CARD's oracle_id and cmc. + built.setdefault("oracle_id", str(row.get("oracle_id") or "")) + built.setdefault("cmc", _decimal(row.get("cmc"))) + if two_image: + built["image_uris"] = _image_uris( + row.get("scryfall_id", ""), + row.get("image_updated_at"), + "front" if index == 0 else "back", + ) + out.append(built) + return out + + +def _decimal(value: float | int | None) -> float | None: + """Carry a mana value as the DECIMAL Scryfall types it as. + + api.scryfall.com answers `"cmc":1.0`, not `"cmc":1` — check + https://api.scryfall.com/cards/named?exact=Lightning+Bolt. The field is decimal because + fractional mana values are real: Little Girl costs {HW} and answers `"cmc":0.5` + (https://api.scryfall.com/cards/named?exact=Little+Girl). A whole-numbered mana value therefore + still serializes with its decimal point, and `magic.cards.cmc` being an `integer` column is what + made this service answer `1` instead. + + That column is also why the underlying 0.5 cannot be stored at all today; changing its type is a + migration and belongs on its own, and nothing that half-mana exists in is imported. This keeps + the SERIALIZATION honest in the meantime, which is what a client comparing against Scryfall + sees. + + Args: + value: The stored mana value, or None. Typed narrowly rather than `Any` because this is the + one place the column's type matters: an `integer` column is exactly what produced the + wrong output. + + Returns: + The value as a float, or None when the card has none. + """ + return None if value is None else float(value) + + +def to_scryfall_card(row: dict[str, Any], *, base_url: str = "https://api.scryfall.com") -> dict[str, Any]: + """Build the Scryfall card object for one engine row. + + BUILDS rather than unwraps a stored copy, which is the whole reason /cards/* can be served from + the engine: an object assembled from columns is answerable from the store, while one recovered + from `raw_card_blob` is answerable only from Postgres — and Postgres is the fallback for when + the engine errors. + + Three sources, and every one of Scryfall's keys comes from exactly one: 29 stored columns, 12 + derived (every *_uri and image_uris, pure functions of the id/set/collector number/oracle id), + and the 33-key residue carried in card_compat_blob. Only `resource_id` is dropped — an + undocumented Scryfall internal with no stable meaning. + + Args: + row: An engine row carrying CARD_OBJECT_FIELDS, or a SQL row through sql_row_to_engine_row. + base_url: The host self-referencing URIs should address. + + Returns: + The card object. Keys Scryfall omits stay omitted rather than becoming null, because a + client comparing shapes would otherwise see a difference on every row. + """ + # str() because a SQL row binds these as UUID objects while an engine row is already a string, + # and every derived URI slices the id. + scryfall_id = str(row.get("scryfall_id") or "") + oracle_id = str(row.get("oracle_id") or "") + name = row.get("name") or "" + set_code = row.get("set_code") or "" + number = row.get("collector_number") or "" + lang = row.get("lang") or "en" + layout = row.get("card_layout") or row.get("layout") + has_faces = bool(row.get("card_faces")) + # Only ever true for a card that HAS faces: the two-image layouts are all multi-face. + two_image = has_faces and layout in _TWO_IMAGE_LAYOUTS + reversible = layout == _REVERSIBLE_LAYOUT + faces = _faces(row, two_image=two_image, reversible=reversible) + # The name a SEARCH LINK spells: the joined one, except on the layouts whose searches take the + # front face (see _JOINED_SEARCH_LAYOUTS). `related_uris.edhrec` and every `purchase_uris` + # fallback take THIS string; the two `tcgplayer_infinite_*` links take the joined `name`. + search_name = name.split(" // ", 1)[0] if faces and layout not in _JOINED_SEARCH_LAYOUTS else name + + card: dict[str, Any] = { + "object": "card", + "id": scryfall_id, + "oracle_id": oracle_id, + "multiverse_ids": row.get("multiverse_ids") or [], + "name": name, + "lang": lang, + "released_at": row.get("released_at"), + "uri": f"{base_url}/cards/{scryfall_id}", + "scryfall_uri": _scryfall_uri(name, set_code, number, lang), + "layout": row.get("card_layout") or row.get("layout"), + "highres_image": bool(row.get("highres_image")), + "image_status": row.get("image_status"), + "cmc": _decimal(row.get("cmc")), + "type_line": row.get("type_line"), + "colors": row.get("colors") or [], + "color_identity": row.get("color_identity") or [], + "keywords": row.get("card_keywords") or [], + "games": row.get("games") or [], + "reserved": "reserved" in (row.get("card_is_tags") or []), + "finishes": row.get("finishes") or [], + "oversized": bool(row.get("oversized")), + "promo": bool(row.get("promo")), + "reprint": bool(row.get("reprint")), + "variation": bool(row.get("variation")), + "set_id": row.get("set_id"), + "set": set_code, + "set_name": row.get("set_name"), + "set_type": row.get("set_type"), + "set_uri": f"{base_url}/sets/{row['set_id']}" if row.get("set_id") else None, + "set_search_uri": f"{base_url}/cards/search?order=set&q=e%3A{set_code}&unique=prints", + "scryfall_set_uri": f"https://scryfall.com/sets/{set_code}?utm_source=api", + "rulings_uri": f"{base_url}/cards/{scryfall_id}/rulings", + "prints_search_uri": f"{base_url}/cards/search?order=released&q=oracleid%3A{oracle_id}&unique=prints", + "collector_number": number, + "digital": bool(row.get("digital")), + "rarity": row.get("rarity"), + "card_back_id": CARD_BACK_ID, + "artist": row.get("artist"), + "illustration_id": str(row["illustration_id"]) if row.get("illustration_id") else None, + "border_color": row.get("border_color"), + "full_art": bool(row.get("full_art")), + "textless": bool(row.get("textless")), + "booster": bool(row.get("booster")), + "story_spotlight": bool(row.get("story_spotlight")), + "prices": _prices(row), + "related_uris": _related_uris(name, search_name, row.get("multiverse_ids") or [], lang), + } + # A printing NO MARKETPLACE SELLS omits the key rather than carrying three dead links. The + # rule is the marketplaces, not `digital` -- measured 2026-08-16: prm/80925 (games ["mtgo"], + # digital true) HAS purchase_uris and ymid/59 and khm/A-198 (games ["arena"], digital true) do + # not, so it is "paper or mtgo". An ABSENT `games` list emits: the omission is a positive claim + # about the printing rather than a gap. + games = row.get("games") + if games is None or not games or any(g in ("paper", "mtgo") for g in games): + card["purchase_uris"] = _purchase_uris(row, search_name) + + # A two-image layout keeps `colors`, `card_back_id` and `illustration_id` on its FACES alone -- + # there is no shared back and no card-level illustration when the card is two pictures -- + # and Scryfall omits the top-level keys entirely rather than nulling them. + if two_image: + for key in ("colors", "card_back_id", "illustration_id"): + card.pop(key, None) + # ...and a reversible printing drops the three the other two-image layouts keep. + if reversible: + for key in ("oracle_id", "cmc", "type_line"): + card.pop(key, None) + + # A multi-face card carries its faces and NOT the top-level text they replace; a single-faced + # one carries the text and no `card_faces`. Which keys sit at top level varies by LAYOUT, which + # is why this is a branch rather than a fixed key set. + if faces: + card["card_faces"] = faces + if not two_image: + # ONE image and one cost: a split/flip/adventure/prepare printing keeps both at top + # level, the cost joined " // " between the faces that HAVE one, skipping the ones + # that do not -- flipped Erayo, whose back face carries an empty cost, is `{1}{U}` and + # not `{1}{U} // `. Checked against all 3,654 such printings with zero misses. + card["mana_cost"] = " // ".join( + f["mana_cost"] for f in (row.get("card_faces") or []) if f.get("mana_cost") + ) + card["image_uris"] = _image_uris(scryfall_id, row.get("image_updated_at")) + else: + card["mana_cost"] = row.get("mana_cost") + card["oracle_text"] = row.get("oracle_text") + card["image_uris"] = _image_uris(scryfall_id, row.get("image_updated_at")) + + # Keys Scryfall sends only when the card has them. Emitting null instead would differ from + # Scryfall on every card that lacks them, which for most of these is most cards. + for key, value in ( + ("power", row.get("power")), + ("toughness", row.get("toughness")), + # Where Scryfall puts it, beside the creature stats it is the planeswalker analogue of. The + # PRINTED string: `planeswalker_loyalty` is a u8 in the engine and cannot hold "X" or "1+*". + ("loyalty", row.get("loyalty")), + ("flavor_text", row.get("flavor_text") or None), + ("watermark", row.get("watermark")), + ("frame", row.get("frame")), + ("edhrec_rank", row.get("edhrec_rank")), + ("penny_rank", row.get("penny_rank")), + ("arena_id", row.get("arena_id")), + ("mtgo_id", row.get("mtgo_id")), + ("mtgo_foil_id", row.get("mtgo_foil_id")), + ("tcgplayer_id", row.get("tcgplayer_id")), + ("tcgplayer_etched_id", row.get("tcgplayer_etched_id")), + ("cardmarket_id", row.get("cardmarket_id")), + ("security_stamp", row.get("security_stamp")), + ("promo_types", row.get("promo_types") or None), + ("frame_effects", row.get("frame_effects") or None), + ("all_parts", row.get("all_parts") or None), + ("legalities", row.get("legalities")), + ): + if value is not None: + card[key] = value + + return card + + +def error_object(*, code: str, status: int, details: str, warnings: list[str] | None = None) -> dict[str, Any]: + """Build Scryfall's error object. + + Args: + code: Scryfall's machine-readable error slug, e.g. "not_found". + status: The HTTP status the response carries. + details: Human-readable explanation. + warnings: Non-fatal notes about the request, when there are any. + + Returns: + The error object, with `warnings` present only when non-empty. + """ + error: dict[str, Any] = {"object": "error", "code": code, "status": status, "details": details} + if warnings: + error["warnings"] = warnings + return error + + +def not_found_error(details: str) -> dict[str, Any]: + """Build the 404 error object. + + Args: + details: Human-readable explanation. + + Returns: + The error object. + """ + return error_object(code="not_found", status=404, details=details) + + +def bad_request_error(details: str, *, warnings: list[str] | None = None) -> dict[str, Any]: + """Build the 400 error object. + + Args: + details: Human-readable explanation. + warnings: Non-fatal notes about the request. + + Returns: + The error object. + """ + return error_object(code="bad_request", status=400, details=details, warnings=warnings) + + +def card_list( # noqa: PLR0913 + cards: list[dict[str, Any]], + *, + total_cards: int | None = None, + has_more: bool = False, + next_page: str | None = None, + not_found: list[dict[str, Any]] | None = None, + warnings: list[str] | None = None, +) -> dict[str, Any]: + """Build Scryfall's List object. + + Key order follows Scryfall's own so a byte-comparing client sees the same document. + + Args: + cards: The page of objects. + total_cards: Unpaginated match count; omitted on lists that do not paginate. + has_more: Whether a further page exists. + next_page: Absolute URL of the next page, when there is one. + not_found: Identifiers a collection request could not resolve. + warnings: Non-fatal notes about the request. + + Returns: + The List object. + """ + result: dict[str, Any] = {"object": "list"} + if total_cards is not None: + result["total_cards"] = total_cards + if not_found is not None: + result["not_found"] = not_found + result["has_more"] = has_more + if next_page is not None: + result["next_page"] = next_page + if warnings: + result["warnings"] = warnings + result["data"] = cards + return result + + +def catalog_object(values: list[str], uri: str | None = None) -> dict[str, Any]: + """Build Scryfall's Catalog object. + + `uri` is present IFF one is given, and the two callers genuinely differ -- measured against + api.scryfall.com on 2026-08-12, `/catalog/battle-types` answers + `{"object": "catalog", "uri": "...", "total_values": 1, "data": ["Siege"]}` while + `/cards/autocomplete` answers the same object with no `uri` at all. Building one unconditionally + would put a key on the autocomplete catalog that Scryfall does not send. + + The uri points at api.scryfall.com rather than at this host, which is the rule the card objects + already follow: a self-referencing URI is part of the payload, not pagination. + + Args: + values: The catalog entries. + uri: The catalog's own URI, for the routes that carry one. + + Returns: + The Catalog object. + """ + catalog: dict[str, Any] = {"object": "catalog"} + if uri is not None: + catalog["uri"] = uri + catalog["total_values"] = len(values) + catalog["data"] = values + return catalog + + +def ruling_object(row: dict[str, Any]) -> dict[str, Any]: + """Build one Scryfall Ruling object from a `magic.rulings` row. + + Args: + row: A row with oracle_id, source, published_at and comment. + + Returns: + The Ruling object. + """ + return { + "object": "ruling", + "oracle_id": str(row["oracle_id"]), + "source": row["source"], + "published_at": row["published_at"].isoformat(), + "comment": row["comment"], + } + + +def build_page_url(base_url: str, params: dict[str, Any], page: int) -> str: + """Build the absolute `next_page` URL for a search result. + + Scryfall spells every effective parameter into `next_page` rather than echoing only what the + client sent, and clients follow the URL verbatim, so the query string is rebuilt from the + resolved values. + + Args: + base_url: Scheme and host the request arrived on, plus the route path. + params: Effective query parameters, excluding `page`. + page: The page number the URL should fetch. + + Returns: + The absolute URL. + """ + query = dict(sorted(params.items())) + query["page"] = page + return f"{base_url}?{urllib.parse.urlencode(sorted(query.items()))}" + + +def _face_of(card: dict[str, Any], face: str) -> dict[str, Any]: + """Return the requested face of a card, falling back to the card itself. + + Args: + card: A Scryfall card object. + face: "back" for the second face; anything else selects the card/front. + + Returns: + The face object, or the card when it has no distinct faces. + """ + faces = card.get("card_faces") or [] + back_face_count = 2 + if face == "back" and len(faces) >= back_face_count: + return faces[1] + return card + + +def image_uri(card: dict[str, Any], *, version: str, face: str) -> str | None: + """Return the image URL for a card at a given size and face. + + Args: + card: A Scryfall card object. + version: One of IMAGE_VERSIONS. + face: "front" or "back". + + Returns: + The image URL, or None when the card carries no image of that size. + """ + selected = _face_of(card, face) + uris = selected.get("image_uris") or card.get("image_uris") or {} + return uris.get(version) + + +def _render_face(face: dict[str, Any]) -> str: + """Render one card face in Scryfall's plain-text format. + + Args: + face: A card or card_face object. + + Returns: + The rendered face, without a trailing newline. + """ + heading = face.get("name", "") + mana_cost = face.get("mana_cost") + if mana_cost: + heading = f"{heading} {mana_cost}" + + lines = [heading] + if face.get("type_line"): + lines.append(face["type_line"]) + if face.get("oracle_text"): + lines.append(face["oracle_text"]) + if face.get("power") is not None and face.get("toughness") is not None: + lines.append(f"{face['power']}/{face['toughness']}") + elif face.get("loyalty") is not None: + lines.append(f"Loyalty: {face['loyalty']}") + elif face.get("defense") is not None: + lines.append(f"Defense: {face['defense']}") + return "\n".join(lines) + + +def card_to_text(card: dict[str, Any]) -> str: + """Render a card in Scryfall's `format=text` layout. + + Args: + card: A Scryfall card object. + + Returns: + The rendered card. Multi-face cards render every face, separated by a blank line. + """ + faces = card.get("card_faces") or [] + if faces: + return "\n\n".join(_render_face(face) for face in faces) + return _render_face(card) diff --git a/api/scryfall_compat/reference_routes.py b/api/scryfall_compat/reference_routes.py new file mode 100644 index 000000000..e06879da9 --- /dev/null +++ b/api/scryfall_compat/reference_routes.py @@ -0,0 +1,382 @@ +"""The Scryfall-compatible `/sets`, `/catalog/*` and `/symbology` routes. + +`ScryfallReferenceRoutes` is a second mixin on `APIResource`, alongside `ScryfallCardsRoutes`. It is +separate because it is a separate kind of thing: these routes answer from the reference tables +mirrored by `api/scryfall_reference_import.py` rather than from the corpus, so none of them touch +the engine, the parser or `_search`. + +The same two conventions as the cards routes apply — every parameter is annotated `str` so the +generic binder never puts a non-Scryfall error body on the wire, and the router matches a full path +before falling back to the first segment, which is what lets `/symbology/parse-mana` claim its exact +path while `/sets/tcgplayer/:id` arrives at `scryfall_sets` as positional segments. + +One route is computed rather than mirrored: `/symbology/parse-mana`, in `mana.py`. +""" + +from __future__ import annotations + +import logging +from typing import Any + +# A runtime import, not a type-checking one, even though `falcon` appears only in annotations here: +# `@route` registration runs every handler's annotations through `typing.get_type_hints`, which +# evaluates them for real. Behind `if TYPE_CHECKING` the name is absent and registration dies with +# `NameError: name 'falcon' is not defined` before the app can serve anything. +import falcon # noqa: TC002 + +from api.scryfall_compat.mana import ManaCostError, parse_mana_cost +from api.scryfall_compat.objects import card_list, catalog_object, error_object, not_found_error +from api.scryfall_compat.responder import ScryfallResponder +from api.utils.routing import route + +logger = logging.getLogger(__name__) + +# Cache tiers, matched to what api.scryfall.com sends on each of these routes (measured +# 2026-08-11). They are not the `public, max-age=57600` the card routes carry: +# +# /sets, /sets/:code, /sets/tcgplayer/:id, /catalog/*, /symbology -> public +# /symbology/parse-mana -> max-age=0, private, must-revalidate +# +# Bare `public` with no max-age leaves freshness to the cache's heuristics, which is weaker than an +# explicit lifetime and is arguably a wart upstream. It is mirrored anyway, because a client that +# swaps its base URL should get the same caching behaviour it tuned against Scryfall — a response +# this service holds for 16 hours where Scryfall revalidates is a behavioural difference the client +# cannot see until it serves something stale. +_MIRRORED_CACHE_CONTROL = "public" + +# parse-mana is the deterministic one, so caching it hard would be safe -- but Scryfall marks it +# private and must-revalidate, and parity is the point. `private` does not defeat this service's own +# CachingMiddleware (only `no-store` does, which is why /cards/random uses that), so the in-process +# cache still answers repeat parses. +_PARSE_MANA_CACHE_CONTROL = "max-age=0, private, must-revalidate" + +# The tier on a MISS THAT IS ABOUT THE ROUTE rather than about Magic, measured 2026-08-16 -- and it +# is a real split, not noise. `/sets/zzzz`, a well-formed set lookup that found nothing, is `public`: +# the same tier the answer would have had, because "there is no such set" is a fact about Magic. +# `/catalog/not-a-catalog`, `/catalog/Card-Types`, `/sets/khm/extra` and every parse-mana 422 are +# `no-cache`, because those are facts about the URL. This surface sent `public` on all of them, so a +# client that mistyped a catalog name got the mistake held at every edge for as long as the +# heuristics liked. +_ROUTE_MISS_CACHE_CONTROL = "no-cache" + +# The twenty catalogs Scryfall documents. Listed rather than discovered so that a request for a name +# this instance has never imported 404s as an unknown catalog, instead of reporting an empty one and +# letting a client conclude Magic has no creature types. +CATALOG_NAMES = ( + "card-names", + "artist-names", + "word-bank", + "supertypes", + "card-types", + "artifact-types", + "battle-types", + "creature-types", + "enchantment-types", + "land-types", + "planeswalker-types", + "spell-types", + "powers", + "toughnesses", + "loyalties", + "watermarks", + "keyword-abilities", + "keyword-actions", + "ability-words", + "flavor-words", +) + +# Scryfall's own not-found wording on these routes, measured against api.scryfall.com on +# 2026-08-12. Neither is the generic body the cards surface sends, and the catalog one is that +# sentence WITHOUT its "Please double-check your URI and try again." tail -- same sentence, +# different ending, so they are spelled out separately rather than shared. +# +# /sets/, /sets/, /sets/tcgplayer/, /sets/tcgplayer +# "No Magic set found for the given code or ID" +# /catalog/ +# "The requested object or REST method was not found." +_SET_MISS_DETAILS = "No Magic set found for the given code or ID" +_CATALOG_MISS_DETAILS = "The requested object or REST method was not found." +# The wording for a path that addresses nothing. Today it is the same sentence the catalog miss uses +# and it is spelled separately anyway: the two mean different things ("there is no such catalog" +# against "there is no such route"), and one of them changing must not be blocked by the other. +_ROUTE_MISS_DETAILS = "The requested object or REST method was not found." + +# The host a Catalog's own `uri` points at: Scryfall's, not this service's, which is the rule the +# card objects already follow for `uri`, `rulings_uri` and `prints_search_uri`. +_SCRYFALL_API = "https://api.scryfall.com" + +# Path segment naming the external-id namespace under /sets, mirroring the /cards namespaces. +_SETS_TCGPLAYER_NAMESPACE = "tcgplayer" + + +def _as_bool(value: str | None, *, default: bool = False) -> bool: + """Parse a Scryfall boolean query parameter. + + Args: + value: The raw parameter value, or None when absent. + default: What an absent parameter means. + + Returns: + The parsed flag; anything other than a recognized true spelling is False. + """ + if value is None: + return default + return value.strip().lower() in ("1", "true", "yes", "on") + + +def _set_reference_cache(falcon_response: falcon.Response | None, tier: str = _MIRRORED_CACHE_CONTROL) -> None: + """Set a reference-route cache tier. + + Args: + falcon_response: The response to write to, or None for an internal caller. + tier: The Cache-Control value; defaults to the tier the mirrored routes share. + """ + if falcon_response is not None: + falcon_response.set_header("Cache-Control", tier) + + +class ScryfallReferenceRoutes(ScryfallResponder): + """The `/sets`, `/catalog` and `/symbology` routes, mixed into `APIResource`. + + Depends on `_run_query` and `_require_setup_complete` from the class it is mixed into, the same + way `ScryfallCardsRoutes` does. + """ + + # ---------------------------------------------------------------- GET /sets + + @route(paths=("sets",)) + def scryfall_sets( + self, + identifier: str = "", + second: str = "", + *, + falcon_response: falcon.Response | None = None, + pretty: str = "false", + **_: object, + ) -> dict[str, Any] | None: + """Answer every `/sets` shape. + + Covers `/sets`, `/sets/:code`, `/sets/:id` and `/sets/tcgplayer/:id` — one handler because + the router hands trailing segments to whichever route claims the first one. + + Args: + identifier: A set code, a Scryfall set id, or the "tcgplayer" namespace. + second: The TCGplayer id, when `identifier` named that namespace. + falcon_response: The Falcon response to write to. + pretty: Whether to indent JSON output. + + Returns: + A List object of sets, one Set object, or a Scryfall error. + """ + is_pretty = _as_bool(pretty) + _set_reference_cache(falcon_response) + self._require_setup_complete() + + if not identifier: + return self._scryfall_respond(falcon_response, card_list(self._all_sets()), pretty=is_pretty) + + if identifier.lower() == _SETS_TCGPLAYER_NAMESPACE: + if not second: + # Scryfall answers the namespace-with-no-id path with its ordinary set miss, not + # with a message about the id being absent. Clearer is not the goal here. + return self._scryfall_respond(falcon_response, not_found_error(_SET_MISS_DETAILS), pretty=is_pretty) + found = self._set_by_tcgplayer_id(second) + elif second: + # /sets takes at most one identifying segment; anything longer addresses nothing -- and + # that is a statement about the URL, not about Magic, so it answers with the ROUTE miss + # rather than the set one. `/sets/khm/extra` on api.scryfall.com is "The requested object + # or REST method was not found." at `no-cache`, not "No Magic set found ..." at `public` + # (measured 2026-08-16); this sent the latter, which told a client the set was missing + # when the set was fine and the path was not. + _set_reference_cache(falcon_response, _ROUTE_MISS_CACHE_CONTROL) + return self._scryfall_respond(falcon_response, not_found_error(_ROUTE_MISS_DETAILS), pretty=is_pretty) + else: + found = self._set_by_code_or_id(identifier) + + if found is None: + return self._scryfall_respond(falcon_response, not_found_error(_SET_MISS_DETAILS), pretty=is_pretty) + return self._scryfall_respond(falcon_response, found, pretty=is_pretty) + + def _all_sets(self) -> list[dict[str, Any]]: + """Every set, in the order Scryfall returns them. + + Returns: + The Set objects. + """ + rows = self._run_query( + query="SELECT set_object FROM magic.sets ORDER BY position", + params={}, + explain=False, + )["result"] + return [row["set_object"] for row in rows] + + def _set_by_code_or_id(self, identifier: str) -> dict[str, Any] | None: + """One set by set code or by Scryfall set id. + + A single query over both keys rather than a UUID test first: a set code is never shaped like + a UUID, so the two can never both match, and one round trip answers either spelling. + + Args: + identifier: The set code or set id. + + Returns: + The Set object, or None when nothing matches. + """ + rows = self._run_query( + query=("SELECT set_object FROM magic.sets WHERE lower(code) = %(folded)s OR id::text = %(raw)s LIMIT 1"), + params={"folded": identifier.lower(), "raw": identifier.lower()}, + explain=False, + )["result"] + return rows[0]["set_object"] if rows else None + + def _set_by_tcgplayer_id(self, raw_id: str) -> dict[str, Any] | None: + """One set by its TCGplayer group id. + + Args: + raw_id: The id as it appeared in the path. + + Returns: + The Set object, or None when the id is unparseable or matches nothing. + """ + try: + tcgplayer_id = int(raw_id.strip()) + except (ValueError, AttributeError): + return None + rows = self._run_query( + query="SELECT set_object FROM magic.sets WHERE tcgplayer_id = %(value)s LIMIT 1", + params={"value": tcgplayer_id}, + explain=False, + )["result"] + return rows[0]["set_object"] if rows else None + + # ---------------------------------------------------------------- GET /catalog/:name + + @route(paths=("catalog",)) + def scryfall_catalog( + self, + name: str = "", + *, + falcon_response: falcon.Response | None = None, + pretty: str = "false", + **_: object, + ) -> dict[str, Any] | None: + """Return one catalog. + + Args: + name: The catalog name, e.g. "creature-types". + falcon_response: The Falcon response to write to. + pretty: Whether to indent JSON output. + + Returns: + A Catalog object, or a Scryfall error. + """ + is_pretty = _as_bool(pretty) + _set_reference_cache(falcon_response) + self._require_setup_complete() + + # VERBATIM, not lowercased and not stripped: catalog names are CASE-SENSITIVE on + # api.scryfall.com -- `/catalog/Card-Types` is a 404 there and was a 200 here (measured + # 2026-08-16). Folding the case made this route answer a URL Scryfall does not serve, which + # is the same class of mistake as failing to answer one it does. + wanted = name + if wanted not in CATALOG_NAMES: + # `no-cache`, not the data tier: a 404 about the PATH is a statement about the URL, and + # Scryfall declines to cache those. Its `/sets/zzzz` -- a well-formed set lookup that + # found nothing -- keeps `public`, which is the other half of the same rule. + _set_reference_cache(falcon_response, _ROUTE_MISS_CACHE_CONTROL) + return self._scryfall_respond(falcon_response, not_found_error(_CATALOG_MISS_DETAILS), pretty=is_pretty) + + rows = self._run_query( + query="SELECT entries FROM magic.catalogs WHERE name = %(name)s", + params={"name": wanted}, + explain=False, + )["result"] + # A known catalog that has not been imported yet is empty rather than missing: the name is + # real, so 404 would tell a client the endpoint does not exist. + entries = rows[0]["entries"] if rows else [] + return self._scryfall_respond( + falcon_response, + catalog_object(list(entries), uri=f"{_SCRYFALL_API}/catalog/{wanted}"), + pretty=is_pretty, + ) + + # ---------------------------------------------------------------- GET /symbology + + @route(paths=("symbology",)) + def scryfall_symbology( + self, + *, + falcon_response: falcon.Response | None = None, + pretty: str = "false", + **_: object, + ) -> dict[str, Any] | None: + """Return every card symbol. + + Args: + falcon_response: The Falcon response to write to. + pretty: Whether to indent JSON output. + + Returns: + A List object of CardSymbol objects. + """ + is_pretty = _as_bool(pretty) + _set_reference_cache(falcon_response) + self._require_setup_complete() + + rows = self._run_query( + query="SELECT symbol_object FROM magic.card_symbols ORDER BY position", + params={}, + explain=False, + )["result"] + return self._scryfall_respond( + falcon_response, + card_list([row["symbol_object"] for row in rows]), + pretty=is_pretty, + ) + + # ---------------------------------------------------------------- GET /symbology/parse-mana + + @route(paths=("symbology/parse-mana",)) + def scryfall_parse_mana( + self, + *, + falcon_response: falcon.Response | None = None, + cost: str | None = None, + pretty: str = "false", + **_: object, + ) -> dict[str, Any] | None: + """Parse a mana cost into Scryfall's ManaCost object. + + The one reference route that reads no table: it is a pure function of `cost`, so it answers + the same before the first import as after it. + + Args: + falcon_response: The Falcon response to write to. + cost: The mana cost as written. + pretty: Whether to indent JSON output. + + Returns: + A ManaCost object, or a Scryfall error. + """ + is_pretty = _as_bool(pretty) + _set_reference_cache(falcon_response, _PARSE_MANA_CACHE_CONTROL) + + # A MISSING `cost` is the same request as an empty one, and both are answered: measured + # 2026-08-16, `/symbology/parse-mana` with no parameter and `?cost=` both return + # `200 {"object": "mana_cost", "cost": null, "colors": [], "cmc": 0.0, ...}`. This sent a 400 + # saying "You must provide a cost parameter to parse." -- a sentence Scryfall does not own + # and a rejection it does not make. `parse_mana_cost("")` already produces exactly that body. + try: + parsed = parse_mana_cost(cost or "") + except ManaCostError as bad_cost: + # 422 rather than 400, which is what Scryfall answers an unparseable fragment with -- at + # `no-cache` rather than this route's own tier, because an unreadable cost is a fact + # about the REQUEST and Scryfall declines to cache those (measured on `{QQQ}`, `!!!`, + # `{}`, `é` and `{W/U/B}`). + _set_reference_cache(falcon_response, _ROUTE_MISS_CACHE_CONTROL) + return self._scryfall_respond( + falcon_response, + error_object(code="validation_error", status=422, details=str(bad_cost)), + pretty=is_pretty, + ) + return self._scryfall_respond(falcon_response, parsed, pretty=is_pretty) diff --git a/api/scryfall_compat/responder.py b/api/scryfall_compat/responder.py new file mode 100644 index 000000000..168f58d1e --- /dev/null +++ b/api/scryfall_compat/responder.py @@ -0,0 +1,72 @@ +"""Response plumbing shared by every Scryfall-compatible surface. + +The `/cards/*` routes and the reference routes (`/sets`, `/catalog`, `/symbology`) are separate +mixins on `APIResource`, but they answer with the same content type, the same `pretty` handling and +the same rule for errors: a handler returns a Scryfall error object rather than raising, and the +status rides on the payload so the generic Falcon error serializer never sees it. That rule lives +here so both surfaces cannot drift apart on it. +""" + +from __future__ import annotations + +from typing import Any + +import falcon +import falcon.util +import orjson + +# Spelled out rather than falcon.MEDIA_JSON, which omits the charset Scryfall sends. +JSON_CONTENT_TYPE = "application/json; charset=utf-8" + + +class ScryfallResponder: + """Writes Scryfall-shaped JSON, honoring the status an error object carries.""" + + def _scryfall_respond( + self, + falcon_response: falcon.Response | None, + payload: dict[str, Any], + *, + pretty: bool = False, + ) -> dict[str, Any] | None: + """Write a JSON payload, honoring the error status it carries and `pretty`. + + Args: + falcon_response: The response to write to. + payload: The Scryfall object to serialize. + pretty: Whether to emit indented JSON. + + Returns: + The payload when the caller should let the framework serialize it, or None when this + method already wrote the body. + """ + if falcon_response is not None: + falcon_response.content_type = JSON_CONTENT_TYPE + is_error = payload.get("object") == "error" + status = payload.get("status") if is_error else None + if isinstance(status, int): + falcon_response.status = falcon.util.code_to_http_status(status) + # AN ERROR BODY IS ALWAYS INDENTED, whatever `pretty` says. Not a style choice: it is + # what api.scryfall.com does. Measured 2026-08-16 across the whole surface -- every + # `object: "error"` body comes back two-space-indented while every data body comes back + # compact, and it does not negotiate: `Accept: application/json`, `Accept: text/html`, a + # bare wildcard and an explicit `?pretty=false` all produce the same 130-byte indented + # not-found. Scryfall renders errors through a different serializer than answers, and + # this rendered both compact, so a client comparing bytes saw a different document for + # every 4xx it received. + if pretty or is_error: + falcon_response.text = orjson.dumps(payload, option=orjson.OPT_INDENT_2).decode() + return None + return payload + + def _respond_text(self, falcon_response: falcon.Response | None, body: str, content_type: str) -> None: + """Write a non-JSON body. + + Args: + falcon_response: The response to write to. + body: The rendered document. + content_type: Its media type. + """ + if falcon_response is not None: + falcon_response.content_type = content_type + falcon_response.text = body diff --git a/api/scryfall_compat/routes.py b/api/scryfall_compat/routes.py new file mode 100644 index 000000000..d2f2172a3 --- /dev/null +++ b/api/scryfall_compat/routes.py @@ -0,0 +1,1800 @@ +"""The Scryfall-compatible `/cards/*` routes. + +`ScryfallCardsRoutes` is a mixin on `APIResource`: it is a separate class only so that the +compatibility layer lands in its own file rather than growing `api_resource.py`, and it depends on +`_search`, `_run_query` and `_require_setup_complete` from the class it is mixed into. + +Two conventions run through every handler here: + +- **Every parameter is annotated `str`.** The generic binder coerces by annotation and raises its + own `400` on a bad value, which would put a non-Scryfall error body on the wire. Parsing the + values in the handler keeps every failure inside the Scryfall error object. +- **The router is prefix-based.** `_resolve_action` matches the full path first and then falls back + to the first segment, so the five named sub-routes (`search`, `named`, `autocomplete`, `random`, + `collection`) register their exact paths and everything else — `/cards`, `/cards/:id`, + `/cards/:code/:number/:lang`, the five external-id namespaces, and the rulings variants — arrives + at `scryfall_cards` as up to three positional segments. + +What is *not* identical to api.scryfall.com is recorded in +docs/issues/local-scryfall-cards-api.md; the short version is that the corpus is a filtered subset +of Scryfall's, so a card this instance never imported 404s here and resolves there. +""" + +from __future__ import annotations + +import csv +import io +import logging +import re +from typing import TYPE_CHECKING, Any + +import falcon + +from api.enums import CardOrdering, PreferOrder, SortDirection, UniqueOn +from api.parsing import generate_sql_query, parse_scryfall_query +from api.parsing.card_query_nodes import fold_accents +from api.parsing.query_budget import ( + InvalidRegexPatternError, + QueryBudgetExceeded, + bounded_query_log_context, +) +from api.scryfall_compat import objects +from api.scryfall_compat.objects import ( + CARD_OBJECT_FIELDS, + DEFAULT_IMAGE_VERSION, + IMAGE_VERSIONS, + MAX_AUTOCOMPLETE_VALUES, + MAX_COLLECTION_IDENTIFIERS, + PAGE_SIZE, + bad_request_error, + card_list, + card_to_text, + catalog_object, + error_object, + not_found_error, + ruling_object, + sql_row_to_engine_row, + to_scryfall_card, +) +from api.scryfall_compat.responder import ScryfallResponder +from api.settings import settings +from api.utils import db_utils +from api.utils.routing import route + +if TYPE_CHECKING: + from collections.abc import Callable, Sequence + +logger = logging.getLogger(__name__) + +# Columns every card lookup needs: the blob `to_scryfall_card` reads, plus the id it is re-sorted +# by when a batch comes back in an order the caller did not ask for. +# The columns a card object is built from. Deliberately NOT raw_card_blob: there is one builder +# (objects.to_scryfall_card) and both paths go through it, so the fallback cannot answer differently +# from the engine. The blob is also no longer the card for a multi-face row -- it is the front face +# -- so building from it would silently degrade exactly the cards the merge was written to fix. +_CARD_COLUMNS = ( + "scryfall_id, oracle_id, card_name, card_layout, mana_cost_text, cmc, type_line, oracle_text, " + "creature_power_text AS power, creature_toughness_text AS toughness, card_colors, " + "card_color_identity, card_keywords, card_set_code, set_name, collector_number, " + "card_rarity_int, flavor_text, card_artist AS artist, illustration_id, released_at, " + "card_legalities, card_border, card_watermark, card_frame_data, card_is_tags, " + "card_compat_blob, card_faces" +) + +# ------------------------------------------------------------------ the by-name key rule +# +# `named?exact=` and a `POST /cards/collection` `{"name"}` identifier are two lookups over one set +# of keys, and they are NOT the same lookup. Measured against api.scryfall.com on 2026-08-31, ONE +# IDENTIFIER PER REQUEST -- a collection response's `data` is not in identifier order, and a batched +# probe silently attributes its answers to the wrong needles: +# +# {"name":"Delver of Secrets"} -> Delver of Secrets // Insectile Aberration +# {"name":"Insectile Aberration"} -> the same card (a BACK face names it) +# {"name":"Delver of Secrets // Insectile ..."} -> not_found <- `exact=` answers the card +# {"name":"Fire // Ice"} -> not_found +# {"name":"Wear // Tear"} -> not_found +# {"name":"Bonecrusher Giant // Stomp"} -> not_found +# {"name":"Who // What // When // Where // Why"} -> und/75 (a FIVE-part name IS a key) +# {"name":"Who"} -> not_found (so is `exact=Who`) +# {"name":"Elves"} -> Elves (ffdn/9), the card named that and not +# one of the hundreds containing the word +# {"name":"limduls vault"} -> Lim-Dul's Vault (collated) +# {"name":"Delver of Secrets","set":"mid"} -> mid/47 (set FILTERS the lookup) +# +# So a card answers to its two FACE names when its name splits in EXACTLY two, and to its whole name +# otherwise -- never both. `exact=` adds the joined name of a two-faced card, and that is the only +# key the two surfaces disagree about. +# +# The ENGINE is the path that ships (`name_key_tier` in card_engine/src/lib.rs). These fragments are +# the SQL fallback saying the same thing, so the two cannot answer differently for a needle either +# of them can reach. + + +def _collate_name(value: str) -> str: + """Collate a name: accent-folded, lowercased, every non-alphanumeric character removed. + + This is what Scryfall compares on both name surfaces. Measured on api.scryfall.com, 2026-08-31: + `exact=delverofsecrets`, `exact=Lightning-Bolt`, `exact=limduls vault`, + `exact=Kongming Sleeping Dragon` and `exact=whowhatwhenwherewhy` all resolve, as do the same + spellings as collection identifiers -- and the folded comparison both routes used before + answered 404 to every one of them. It subsumes trimming: `{"name":" Lightning Bolt "}` + resolves there and did not here, because the collection route compared the string as posted. + + `str.isalnum` per character rather than an ASCII class, matching the engine's + `char::is_alphanumeric`: the value is accent-folded first, so a character still non-ASCII at + this point is one NFKD had no base letter for and must be kept, not dropped. + + Args: + value: A name as the client spelled it. + + Returns: + Its collated form, which is "" for a value carrying no alphanumeric character at all. + """ + return "".join(char for char in fold_accents(value.lower()) if char.isalnum()) + + +def _collated_sql(expr: str) -> str: + """The SQL that collates `expr` the way `_collate_name` collates the needle. + + `[:alnum:]` is the server's character class where the engine uses Rust's + `char::is_alphanumeric`. The two agree over ASCII, which is all `card_name_folded` holds on this + corpus -- it is written by `fold_accents` at import. That is the one place the fallback can + drift from the engine, and it needs a name NFKD cannot reduce to ASCII to do it. + """ + return f"regexp_replace(lower({expr}), '[^[:alnum:]]', '', 'g')" + + +_NAME_FRONT = "split_part(card_name_folded, ' // ', 1)" +_NAME_BACK = "split_part(card_name_folded, ' // ', 2)" + +# EXACTLY two halves, which is the load-bearing word: a name with more of them has no face keys at +# all. `exact=Who`, `exact=What` and `{"name":"Who"}` are each not_found on api.scryfall.com while +# `Who // What // When // Where // Why` answers und/75 on both surfaces -- the five-part name is the +# key and its parts are not. Part 3 being empty is what distinguishes the two cases. +_NAME_SPLITS_IN_TWO = f"({_NAME_BACK} <> '' AND split_part(card_name_folded, ' // ', 3) = '')" + +_FACE_NAME_MATCH = f"(%(collated)s IN ({_collated_sql(_NAME_FRONT)}, {_collated_sql(_NAME_BACK)}))" +_WHOLE_NAME_MATCH = f"({_collated_sql('card_name_folded')} = %(collated)s)" + +# A collection identifier's keys: the faces, or the whole name, never both. +_COLLECTION_NAME_MATCH = f"(CASE WHEN {_NAME_SPLITS_IN_TWO} THEN {_FACE_NAME_MATCH} ELSE {_WHOLE_NAME_MATCH} END)" + +# `exact=`'s keys: the same set, plus the JOINED name of a two-faced card. +_EXACT_NAME_MATCH = f"({_WHOLE_NAME_MATCH} OR ({_NAME_SPLITS_IN_TWO} AND {_FACE_NAME_MATCH}))" + +# A WHOLE-name match beats a FACE match on both surfaces, ahead of prefer_score rather than beside +# it. Without it a needle that is one card's whole name and another's face answers whichever scores +# higher: on this corpus `Lightning Bolt` would resolve "Emeritus of Conflict // Lightning Bolt". +# One expression for both scopes -- a two-faced card matched by a face cannot also carry the needle +# as its whole collated name. +_WHOLE_NAME_FIRST = f"{_WHOLE_NAME_MATCH} DESC, " + +# Path segments that name an external id namespace rather than a set code. +_EXTERNAL_ID_NAMESPACES = ("multiverse", "mtgo", "arena", "tcgplayer", "cardmarket") + +# Blob keys each namespace matches. Scryfall's MTGO and TCGplayer routes each accept two ids — +# the regular printing's and the foil/etched printing's — and both resolve to the same card. +# Scryfall's `order` vocabulary, which `CardOrdering` covers except for the two below. Built from +# the enum rather than listed, so an ordering added there is accepted here without a second edit; +# the extra member that is not Scryfall's (`cubecobra`) is a harmless superset. +_ORDER_MAP: dict[str, CardOrdering] = {str(member): member for member in CardOrdering} + +# The two Scryfall orders with no counterpart. `penny` needs penny_rank lifted out of raw_card_blob +# into a column; `review` is Scryfall-internal with no public input and is not reproducible at all. +# Both fall back to `name`, which is what Scryfall does with an order it does not recognize +# (measured 2026-08-09: it falls back silently), and add a warning saying so. +_SCRYFALL_ONLY_ORDERS = ("penny", "review") + +# Scryfall's `dir` vocabulary. `auto` is not resolved here -- it reaches `_search` as AUTO and is +# folded against the ordering there, so this route and /search agree on what auto means. +_DIRECTION_MAP: dict[str, SortDirection] = { + "asc": SortDirection.ASC, + "desc": SortDirection.DESC, + "auto": SortDirection.AUTO, +} + +_UNIQUE_MAP: dict[str, UniqueOn] = { + "cards": UniqueOn.CARD, + "art": UniqueOn.ARTWORK, + "prints": UniqueOn.PRINTING, +} + +# Scryfall's own wording, down to the typographic apostrophe, so a client that string-matches on +# `details` behaves the same. +_NO_MATCH_DETAILS = ( + "Your query didn’t match any cards. Adjust your search terms or refer to the syntax guide " # noqa: RUF001 + "at https://scryfall.com/docs/syntax" +) +_EMPTY_QUERY_DETAILS = "You didn't enter anything to search for." + +# CSV columns for `format=csv`. Fixed rather than derived from the page's cards so the header does +# not change between pages of one result set. Nested objects are flattened with `_`, matching how +# Scryfall spells `image_uris_normal` and `prices_usd` in its own export. +_CSV_SCALAR_COLUMNS = ( + "object", + "id", + "oracle_id", + "multiverse_ids", + "mtgo_id", + "mtgo_foil_id", + "tcgplayer_id", + "cardmarket_id", + "name", + "lang", + "released_at", + "uri", + "scryfall_uri", + "layout", + "highres_image", + "image_status", + "mana_cost", + "cmc", + "type_line", + "oracle_text", + "power", + "toughness", + "loyalty", + "colors", + "color_identity", + "keywords", + "games", + "reserved", + "foil", + "nonfoil", + "finishes", + "oversized", + "promo", + "reprint", + "variation", + "set_id", + "set", + "set_name", + "set_type", + "set_uri", + "set_search_uri", + "scryfall_set_uri", + "rulings_uri", + "prints_search_uri", + "collector_number", + "digital", + "rarity", + "flavor_text", + "card_back_id", + "artist", + "artist_ids", + "illustration_id", + "border_color", + "frame", + "full_art", + "textless", + "booster", + "story_spotlight", + "edhrec_rank", + "penny_rank", +) +_CSV_NESTED_COLUMNS = ( + ("image_uris", IMAGE_VERSIONS), + ("prices", ("usd", "usd_foil", "usd_etched", "eur", "eur_foil", "tix")), + ("related_uris", ("gatherer", "tcgplayer_infinite_articles", "tcgplayer_infinite_decks", "edhrec")), + ("purchase_uris", ("tcgplayer", "cardmarket", "cardhoarder")), +) + + +class _EngineMiss: + """The engine could not serve this lookup, so the caller should try SQL. + + Distinct from None, which means the engine answered and there is no such card. Collapsing the + two would let an unloaded store 404 a card that exists. + """ + + +_ENGINE_MISS = _EngineMiss() + +# SQL fallback only: the blob subfields each external-id namespace maps to. The engine path uses its +# own index and never reads these. +_EXTERNAL_ID_COLUMNS: dict[str, tuple[str, ...]] = { + "multiverse": ("multiverse_ids",), + "mtgo": ("mtgo_id", "mtgo_foil_id"), + "arena": ("arena_id",), + "tcgplayer": ("tcgplayer_id", "tcgplayer_etched_id"), + "cardmarket": ("cardmarket_id",), +} + +_UUID_RE = re.compile(r"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$") + +# Thresholds for the typo-tolerant stage of `?fuzzy=`. A candidate must score at least the floor, +# and the best must lead the next distinct card name by at least the lead — closer than that and +# the query does not identify either card, so it is `ambiguous` rather than a guess. The floor sits +# deliberately above pg_trgm's default 0.3 similarity_threshold, so the index-assisted `%` +# prefilter always admits a strict superset of what the floor keeps. +FUZZY_SIMILARITY_FLOOR = 0.4 +FUZZY_SIMILARITY_LEAD = 0.05 + +# Returned by the similarity stage when two names are too close to choose between. A distinct +# object rather than a flag so the caller compares with `is` and cannot confuse it with a row. +_AMBIGUOUS: dict[str, Any] = {"ambiguous": True} + +# How long a /cards/* answer may be reused, measured against api.scryfall.com rather than chosen: +# it sends `public, max-age=57600` on search, named, autocomplete and every by-id addressing, and +# the tier rides on its error responses too. These routes sent NO Cache-Control at all, so a CDN in +# front of this service cached none of them -- CachingMiddleware is an internal response cache and +# says nothing to anyone downstream. +# +# `/cards/random` keeps its stronger `no-store` (Scryfall sends `no-cache`): the draw must not be +# replayed by either layer, and no-store is the one that also defeats the internal cache. +_CARDS_CACHE_CONTROL = "public, max-age=57600" + + +def _set_cards_cache(falcon_response: falcon.Response | None) -> None: + """Set the /cards/* cache tier. + + Local rather than `api_resource.set_cache_header`, which is the same line of code: these + routes are a MIXIN ON `APIResource`, so `api_resource` imports this module and importing back + is a circular import that fails at startup. + + Args: + falcon_response: The response to write to, or None for an internal caller. + """ + if falcon_response is not None: + falcon_response.set_header("Cache-Control", _CARDS_CACHE_CONTROL) + + +# Hosts an absolute self-URL should address over plain HTTP. Everything else is assumed to be +# reached over TLS, which is what `next_page` has to say for a client to follow it. +_PLAINTEXT_HOSTS = ("localhost", "127.0.0.1", "0.0.0.0", "[::1]") # noqa: S104 + +# Scryfall's three not-found bodies for these routes, measured against api.scryfall.com on +# 2026-08-12. They are worded by the SHAPE of the path rather than by the outcome, and none of them +# is the single string this used to answer with -- which carried a "Please double-check your URI and +# try again." tail Scryfall does not send, so the generic case was wrong as well as the specific +# ones. +# +# /cards/, /cards/ the path addresses nothing +# /cards/, /cards//, /cards//(/) +# a card miss: the address is well formed +# the rulings variants the same, worded for the routes that take a +# multiverse id too +# +# `&` rather than `and`, and `multiverse ID` appearing only in the rulings one, are both Scryfall's. +_NOT_ADDRESSABLE_DETAILS = "The requested object or REST method was not found." +_CARD_MISS_DETAILS = "No card found with the given ID or set code and collector number." +_RULINGS_MISS_DETAILS = "No card found with the given ID, multiverse ID, or set code & collector number." + + +def _miss_details(identifier: str, number: str, suffix: str) -> str: + """Pick the body a `/cards/...` miss answers with. + + Decided from the segments, not from what the lookup did, because that is how Scryfall words + them: `/cards/nonsense` and `/cards/` are both misses and get + different sentences. + + `/cards//rulings` where x is not an id is the subtle one, and it is measured both ways: + Scryfall reads it as a set code and a collector number that happens to be "rulings", so it + answers the CARD miss rather than the rulings one. + + Args: + identifier: First path segment. + number: Second path segment. + suffix: Third path segment. + + Returns: + The `details` string for the 404. + """ + if not number and not _is_uuid(identifier): + return _NOT_ADDRESSABLE_DETAILS + if (number == "rulings" and _is_uuid(identifier)) or suffix == "rulings": + return _RULINGS_MISS_DETAILS + return _CARD_MISS_DETAILS + + +def _is_uuid(value: str) -> bool: + """Return whether a path segment is shaped like a UUID. + + Args: + value: The segment to test. + + Returns: + True when the segment is a canonical 8-4-4-4-12 UUID. + """ + return bool(_UUID_RE.match(value)) + + +def _as_bool(value: str | None, *, default: bool = False) -> bool: + """Parse a Scryfall boolean query parameter. + + Args: + value: The raw parameter value, or None when absent. + default: What an absent parameter means. + + Returns: + The parsed flag; anything other than a recognized true spelling is False. + """ + if value is None: + return default + return value.strip().lower() in ("1", "true", "yes", "on") + + +def _as_int(value: str | None) -> int | None: + """Parse an integer query parameter or path segment. + + Args: + value: The raw value, or None when absent. + + Returns: + The integer, or None when absent or unparseable. + """ + if value is None: + return None + try: + return int(value.strip()) + except (ValueError, AttributeError): + return None + + +def _self_base_url(request: falcon.Request | None, request_host: str, path: str) -> str: + """Build the absolute URL of a route on this host. + + A `next_page` a client cannot follow is worse than no pagination at all, so the scheme is the + request's own as corrected by `Forwarded` / `X-Forwarded-Proto` — the only signal that knows + about a TLS-terminating proxy, behind which the request itself arrives as plain `http`. A + deployment that terminates TLS in front of this service must send one of those headers, as it + must already for `X-Proxy-Host` to give the right host. + + Guessing `https` from the host name instead was tried and is worse: it silently breaks any + plain-HTTP deployment on a real hostname, which is a configuration this project supports, + to paper over one that is misconfigured. The host only decides when there is no request to + read, which is an internal caller rather than a served request. + + Args: + request: The request being answered, when the handler has one. + request_host: Host the request arrived on. + path: Absolute route path, leading slash included. + + Returns: + The absolute URL, with no query string. + """ + host = request_host or "api.scryfall.com" + if request is not None: + return f"{request.forwarded_scheme}://{host}{path}" + scheme = "http" if host.split(":")[0] in _PLAINTEXT_HOSTS else "https" + return f"{scheme}://{host}{path}" + + +def _flatten_for_csv(card: dict[str, Any]) -> dict[str, Any]: + """Flatten a card object onto the fixed CSV column set. + + Args: + card: A Scryfall card object. + + Returns: + A mapping from column name to cell value; list values are joined on commas. + """ + row: dict[str, Any] = {} + for column in _CSV_SCALAR_COLUMNS: + value = card.get(column) + row[column] = ",".join(str(item) for item in value) if isinstance(value, list) else value + for parent, children in _CSV_NESTED_COLUMNS: + nested = card.get(parent) or {} + for child in children: + row[f"{parent}_{child}"] = nested.get(child) + return row + + +def _csv_columns() -> list[str]: + """Return the CSV header in column order. + + Returns: + Every scalar column followed by the flattened nested columns. + """ + columns = list(_CSV_SCALAR_COLUMNS) + for parent, children in _CSV_NESTED_COLUMNS: + columns.extend(f"{parent}_{child}" for child in children) + return columns + + +def _cards_to_csv(cards: Sequence[dict[str, Any]]) -> str: + """Render a page of cards as CSV. + + Args: + cards: The card objects to render. + + Returns: + The CSV document, header row included. + """ + buffer = io.StringIO() + writer = csv.DictWriter(buffer, fieldnames=_csv_columns(), extrasaction="ignore") + writer.writeheader() + for card in cards: + writer.writerow(_flatten_for_csv(card)) + return buffer.getvalue() + + +class ScryfallCardsRoutes(ScryfallResponder): + """The `/cards/*` routes, mixed into `APIResource`. + + Every handler returns the value that becomes the response body, or None after writing the + response itself (the text and CSV formats, which are not JSON). Errors are returned as + Scryfall error objects with the matching status rather than raised, so the generic Falcon + error serializer never sees them — `ScryfallResponder` is where that rule lives, shared with + the `/sets`, `/catalog` and `/symbology` routes. + """ + + def _render_card( # noqa: PLR0913 + self, + card: dict[str, Any], + *, + falcon_response: falcon.Response | None, + card_format: str, + face: str, + version: str, + pretty: bool, + ) -> dict[str, Any] | None: + """Emit one card in the requested format. + + Args: + card: The Scryfall card object. + falcon_response: The response to write to. + card_format: "json", "text" or "image". + face: "front" or "back". + version: One of IMAGE_VERSIONS. + pretty: Whether JSON output is indented. + + Returns: + The payload to serialize, or None when the body was written here. + + Raises: + falcon.HTTPFound: For `format=image`, redirecting to the image itself. + """ + if card_format == "text": + self._respond_text(falcon_response, card_to_text(card), "text/plain; charset=utf-8") + return None + if card_format == "image": + location = objects.image_uri(card, version=version, face=face) + if not location: + return self._scryfall_respond( + falcon_response, + not_found_error("No image is available for this card in that version."), + pretty=pretty, + ) + raise falcon.HTTPFound(location) + return self._scryfall_respond(falcon_response, card, pretty=pretty) + + # ---------------------------------------------------------------- card lookups + + def _run_uncached(self, *, query: str, params: dict[str, Any]) -> list[dict[str, Any]]: + """Run a query that must not be memoized, and return its rows. + + `_run_query` keys its cache on the SQL text and the bound parameters, which is right for + every lookup here except the random draw: that one is deliberately non-deterministic for a + fixed query and parameter set, so caching it would replay one card forever. + + Args: + query: The SQL to run. + params: Bound parameters. + + Returns: + The result rows. + """ + with self.app_context.reader_pool.connection() as conn, conn.cursor() as cursor: + db_utils.set_statement_timeout(cursor, 10_000) + cursor.execute(query, params) + return [dict(row) for row in cursor.fetchall()] + + def _engine_for_lookup(self) -> object | None: + """The engine when it can answer, or None when the caller must fall back to SQL. + + Mirrors the three branches `search()` already uses: feature-gated off, store not loaded, or + ready. SQL is the fallback for when the engine cannot serve, not a peer path -- every route + below asks here first and only reaches Postgres if this returns None or the engine raises. + """ + if not settings.enable_engine: + return None + try: + if self.app_context.engine.size() == 0: + self._trigger_background_reload_if_needed() + return None + # An engine that cannot report its size cannot serve, whatever the reason. + except Exception: # noqa: BLE001 + return None + return self.app_context.engine + + def _engine_card(self, fetch: Callable[[Any], dict[str, Any] | None]) -> dict[str, Any] | _EngineMiss | None: + """Run one engine lookup, or report that the engine could not serve it. + + Returns the card, None for a genuine "no such card", or _ENGINE_MISS when the caller should + try SQL. Separating the last from the first matters: a store that is not loaded must not + answer 404 for a card that exists. + """ + engine = self._engine_for_lookup() + if engine is None: + return _ENGINE_MISS + try: + row = fetch(engine) + # Any engine failure is a fallback, never a 500. + except Exception: + logger.exception("Engine lookup failed, falling back to SQL") + return _ENGINE_MISS + return to_scryfall_card(row) if row else None + + def _card_by_scryfall_id(self, scryfall_id: str) -> dict[str, Any] | None: + """One card by Scryfall id, from the store when it can answer.""" + found = self._engine_card(lambda e: e.card_by_scryfall_id(str(scryfall_id), list(CARD_OBJECT_FIELDS))) + if found is not _ENGINE_MISS: + return found + return self._fetch_one_card("scryfall_id = %(value)s", {"value": str(scryfall_id)}) + + def _card_by_oracle_id(self, oracle_id: str) -> dict[str, Any] | None: + """The representative printing of one oracle card, from the store when it can answer.""" + engine = self._engine_for_lookup() + if engine is not None: + try: + rows = engine.printings_of_oracle_id(str(oracle_id), list(CARD_OBJECT_FIELDS)) + # Printings are stored in descending default-prefer order, so the first is the + # representative printing every other by-name path shows. + return to_scryfall_card(rows[0]) if rows else None + except Exception: + logger.exception("Engine oracle-id lookup failed, falling back to SQL") + return self._fetch_one_card("oracle_id = %(value)s", {"value": str(oracle_id)}) + + def _card_by_external_id(self, namespace: str, external_id: int | None) -> dict[str, Any] | None: + """One card by a marketplace or client id, from the store when it can answer.""" + if external_id is None: + return None + found = self._engine_card( + lambda e: e.card_by_external_id(namespace, int(external_id), list(CARD_OBJECT_FIELDS)), + ) + if found is not _ENGINE_MISS: + return found + columns = _EXTERNAL_ID_COLUMNS.get(namespace, ()) + if not columns: + return None + clauses = " OR ".join(f"(raw_card_blob ->> '{column}')::bigint = %(value)s" for column in columns) + return self._fetch_one_card(f"({clauses})", {"value": external_id}) + + def _fetch_one_card(self, where: str, params: dict[str, Any], *, rank_first: str = "") -> dict[str, Any] | None: + """Fetch the single best printing matching a predicate. + + Ties are broken by prefer_score, so a lookup that spans printings (by name, by oracle id) + returns the same representative printing the rest of the API would pick. + + Args: + where: SQL predicate, referencing `card` as the table alias. + params: Bound parameters for the predicate. + rank_first: An ORDER BY term applied BEFORE prefer_score, for a caller whose predicate + admits matches of different qualities. `named?exact=` needs it: it matches either + face of a "Front // Back" name, and without this a two-faced card whose back face + carries the name outranks the card actually named that whenever its score is + higher. + + Returns: + The card, or None when nothing matched. + """ + rows = self._run_query( + query=( + f"SELECT {_CARD_COLUMNS} FROM magic.cards AS card WHERE {where} " + f"ORDER BY {rank_first}prefer_score DESC NULLS LAST, released_at DESC LIMIT 1" + ), + params=params, + explain=False, + )["result"] + return to_scryfall_card(sql_row_to_engine_row(rows[0])) if rows else None + + def _engine_exact_name(self, folded: str, set_code: str | None) -> dict[str, Any] | None: + """The exact-name match from the engine, or None when it cannot answer. + + Args: + folded: The accent-folded, lowercased name. + set_code: Restrict to this set, or None for any. + + Returns: + `{"scryfall_id", "card_name"}` for the best match, or None to fall back to SQL. + """ + engine = self._engine_for_lookup() + if engine is None: + return None + try: + row = engine.exact_card_by_name(folded, set_code, list(CARD_OBJECT_FIELDS)) + # Any engine failure falls back to SQL; it never 500s. + except Exception: + logger.exception("Engine exact name match failed, falling back to SQL") + return None + if row is None: + return None + # Outside the except ON PURPOSE: a missing key is a SHAPE mismatch between this call and + # CARD_OBJECT_FIELDS, not an engine that cannot answer, and swallowing it would turn the + # fast path into a permanent silent fallback. See _fuzzy_similarity_candidate, where + # exactly that happened. + return {"scryfall_id": row["scryfall_id"], "card_name": row["name"]} + + def _card_by_illustration_id(self, illustration_id: str) -> dict[str, Any] | None: + """Return the best printing carrying an illustration id. + + The ENGINE first, like every other identifier `/cards/collection` accepts. This was the one + left on SQL, and it is a scan there too — `illustration_id` has no index on the table, where + the engine answers it from a sorted permutation in O(log n). + + Args: + illustration_id: The illustration UUID. + + Returns: + The matching printing, or None. + """ + engine = self._engine_for_lookup() + if engine is not None: + try: + row = engine.card_by_illustration_id(illustration_id, list(CARD_OBJECT_FIELDS)) + # Any engine failure falls back to SQL; it never 500s. + except Exception: + logger.exception("Engine illustration lookup failed, falling back to SQL") + else: + if row is None: + return None + return self._fetch_one_card("scryfall_id = %(value)s", {"value": str(row["scryfall_id"])}) + + return self._fetch_one_card("illustration_id = %(value)s", {"value": illustration_id}) + + def _cards_by_ids(self, scryfall_ids: Sequence[str]) -> list[dict[str, Any]]: + """Fetch cards by scryfall id, preserving the order of the ids given. + + Args: + scryfall_ids: The ids to fetch. + + Returns: + The cards, in `scryfall_ids` order; ids that matched nothing are skipped. + """ + if not scryfall_ids: + return [] + engine = self._engine_for_lookup() + if engine is not None: + try: + by_id_engine = {} + for card_id in scryfall_ids: + row = engine.card_by_scryfall_id(str(card_id), list(CARD_OBJECT_FIELDS)) + if row: + by_id_engine[str(card_id)] = to_scryfall_card(row) + return [by_id_engine[i] for i in scryfall_ids if i in by_id_engine] + # Hydration failure falls back; it does not 500. + except Exception: + logger.exception("Engine hydration failed, falling back to SQL") + rows = self._run_query( + # A comma-joined string rather than a list: _run_query passes list parameters through + # maybe_json(), which binds them as jsonb, and jsonb does not cast to uuid[]. + query=( + f"SELECT {_CARD_COLUMNS} FROM magic.cards AS card WHERE scryfall_id = ANY(string_to_array(%(ids)s, ',')::uuid[])" + ), + params={"ids": ",".join(scryfall_ids)}, + explain=False, + )["result"] + by_id = {str(row["scryfall_id"]): to_scryfall_card(sql_row_to_engine_row(row)) for row in rows} + return [by_id[card_id] for card_id in scryfall_ids if card_id in by_id] + + # ---------------------------------------------------------------- GET /cards/search + + @route(paths=("cards/search",)) + def scryfall_cards_search( # noqa: PLR0913 + self, + *, + falcon_response: falcon.Response | None = None, + request: falcon.Request | None = None, + request_host: str = "", + q: str | None = None, + unique: str = "cards", + order: str = "name", + dir: str = "auto", # noqa: A002 -- Scryfall's parameter name + page: str = "1", + format: str = "json", # noqa: A002 -- Scryfall's parameter name + pretty: str = "false", + include_extras: str = "false", + include_multilingual: str = "false", + include_variations: str = "false", + **_: object, + ) -> dict[str, Any] | None: + """Search for cards, paginated 175 at a time. + + `include_extras`, `include_multilingual` and `include_variations` are accepted and have no + effect: the corpus holds no tokens, emblems or funny-set cards for `include_extras` to add, + and it holds every printing and language it has imported unconditionally. + + Args: + falcon_response: The Falcon response to write to. + request: The Falcon request, read for the scheme `next_page` should use. + request_host: Host the request arrived on, used to build `next_page`. + q: The search query. + unique: Rollup mode -- cards, art or prints. + order: Sort key. + dir: Sort direction -- auto, asc or desc. + page: 1-based page number. + format: Response format -- json or csv. + pretty: Whether to indent JSON output. + include_extras: Accepted, ignored. + include_multilingual: Accepted, ignored. + include_variations: Accepted, ignored. + + Returns: + A List object of cards, or a Scryfall error object. + """ + is_pretty = _as_bool(pretty) + # Before the handler runs, so the tier rides on the 400s raised inside it too -- which is + # what api.scryfall.com does (an empty-query 400 comes back with the route's own max-age). + _set_cards_cache(falcon_response) + if not q or not q.strip(): + return self._scryfall_respond(falcon_response, bad_request_error(_EMPTY_QUERY_DETAILS), pretty=is_pretty) + + # `or 1` would swallow page=0 into page=1; an unparseable page defaults, a non-positive + # one is rejected below. + parsed_page = _as_int(page) + page_number = 1 if parsed_page is None else parsed_page + if page_number < 1: + return self._scryfall_respond( + falcon_response, + bad_request_error("The page parameter must be a positive integer."), + pretty=is_pretty, + ) + + warnings: list[str] = [] + unique_on = _UNIQUE_MAP.get(unique.lower()) + if unique_on is None: + warnings.append(f"Unrecognized unique mode {unique!r}; rolled up by card instead.") + unique_on = UniqueOn.CARD + + orderby = _ORDER_MAP.get(order.lower()) + if orderby is None: + if order.lower() in _SCRYFALL_ONLY_ORDERS: + warnings.append(f"This server cannot sort by {order!r} yet; sorted by name instead.") + else: + warnings.append(f"Unrecognized order {order!r}; sorted by name instead.") + orderby = CardOrdering.NAME + + # An unrecognized direction falls back to AUTO, which is also the parameter's default -- + # Scryfall ignores one it does not know rather than erroring. + direction = _DIRECTION_MAP.get(dir.lower(), SortDirection.AUTO) + + try: + result = self._search( + query=q, + orderby=orderby, + direction=direction, + fields=["scryfall_id"], + limit=PAGE_SIZE, + offset=(page_number - 1) * PAGE_SIZE, + unique=unique_on, + prefer=PreferOrder.DEFAULT, + ) + except falcon.HTTPBadRequest as err: + return self._scryfall_respond( + falcon_response, + bad_request_error(str(err.description or "The query could not be parsed."), warnings=warnings), + pretty=is_pretty, + ) + + warnings.extend(result.get("warnings") or []) + total_cards = result["total_cards"] + cards = self._cards_by_ids([str(row["scryfall_id"]) for row in result["cards"]]) + if not cards: + return self._scryfall_respond( + falcon_response, + error_object(code="not_found", status=404, details=_NO_MATCH_DETAILS, warnings=warnings), + pretty=is_pretty, + ) + + has_more = (page_number - 1) * PAGE_SIZE + len(cards) < total_cards + next_page = None + if has_more: + next_page = objects.build_page_url( + _self_base_url(request, request_host, "/cards/search"), + { + "dir": dir, + "format": format, + "include_extras": str(_as_bool(include_extras)).lower(), + "include_multilingual": str(_as_bool(include_multilingual)).lower(), + "include_variations": str(_as_bool(include_variations)).lower(), + "order": order, + "q": q, + "unique": unique, + }, + page_number + 1, + ) + + if format.lower() == "csv": + self._respond_text(falcon_response, _cards_to_csv(cards), "text/csv; charset=utf-8") + return None + return self._scryfall_respond( + falcon_response, + card_list(cards, total_cards=total_cards, has_more=has_more, next_page=next_page, warnings=warnings), + pretty=is_pretty, + ) + + # ---------------------------------------------------------------- GET /cards/named + + @route(paths=("cards/named",)) + def scryfall_cards_named( # noqa: PLR0913 + self, + *, + falcon_response: falcon.Response | None = None, + exact: str | None = None, + fuzzy: str | None = None, + set: str | None = None, # noqa: A002 -- Scryfall's parameter name + format: str = "json", # noqa: A002 -- Scryfall's parameter name + face: str = "front", + version: str = DEFAULT_IMAGE_VERSION, + pretty: str = "false", + **_: object, + ) -> dict[str, Any] | None: + """Return one card by exact or fuzzy name. + + Args: + falcon_response: The Falcon response to write to. + exact: A name to match exactly, ignoring case. + fuzzy: A name to match loosely. + set: Restrict the search to one set code. + format: Response format -- json, text or image. + face: Which face an image request wants. + version: Which image size an image request wants. + pretty: Whether to indent JSON output. + + Returns: + A card object, or a Scryfall error object. + """ + is_pretty = _as_bool(pretty) + _set_cards_cache(falcon_response) + self._require_setup_complete() + if not (exact or fuzzy): + return self._scryfall_respond( + falcon_response, + bad_request_error("You must provide a fuzzy or exact name parameter."), + pretty=is_pretty, + ) + + params: dict[str, Any] = {} + clauses = [] + if set: + clauses.append("lower(card_set_code) = lower(%(set_code)s)") + params["set_code"] = set + + if exact: + # Scryfall's exact match ignores case, diacritics AND punctuation, and matches a single + # face of a two-faced card as well as the combined "Front // Back" this corpus stores. + # `_EXACT_NAME_MATCH` is that key set; the block above it carries the measurements, and + # the two things it corrects here are that the comparison is COLLATED rather than + # folded, and that a face key exists only when the name splits in EXACTLY two -- + # `split_part(..., 2)` read *Who // What // When // Where // Why* as having a back face + # named "What", so `exact=What` answered und/75 where Scryfall 404s. + params["folded"] = fold_accents(exact.strip().lower()) + params["collated"] = _collate_name(exact) + clauses.append(_EXACT_NAME_MATCH) + # The ENGINE first, same as the fuzzy stages below and `_cards_by_ids`. This was the + # last by-name lookup still answering from SQL, and it is the one a scan hurts most: + # `named?exact=` is a single-card fetch that walked all ~31,700 folded names. It takes + # the FOLDED needle and collates it itself, so the two paths cannot collate differently. + card = None + chosen = self._engine_exact_name(params["folded"], params.get("set_code")) + if chosen is not None: + found = self._cards_by_ids([str(chosen["scryfall_id"])]) + card = found[0] if found else None + if card is None: + card = self._fetch_one_card(" AND ".join(clauses), params, rank_first=_WHOLE_NAME_FIRST) + if card is None: + return self._scryfall_respond( + falcon_response, + not_found_error(f"No cards found matching “{exact}”"), + pretty=is_pretty, + ) + return self._render_card( + card, + falcon_response=falcon_response, + card_format=format.lower(), + face=face, + version=version, + pretty=is_pretty, + ) + + return self._named_fuzzy( + fuzzy or "", + base_clauses=clauses, + base_params=params, + falcon_response=falcon_response, + card_format=format.lower(), + face=face, + version=version, + pretty=is_pretty, + ) + + def _named_fuzzy( # noqa: PLR0913 + self, + fuzzy: str, + *, + base_clauses: list[str], + base_params: dict[str, Any], + falcon_response: falcon.Response | None, + card_format: str, + face: str, + version: str, + pretty: bool, + ) -> dict[str, Any] | None: + """Resolve a fuzzy name: exact, then all-words-present, then typo-tolerant similarity. + + The three stages mirror what Scryfall resolves in practice — `lightning bolt` exactly, + `bolt` by containment, `lighning bolt` by trigram distance — and each stage that finds more + than one distinct card name reports `ambiguous` rather than guessing between them. + + Args: + fuzzy: The name fragment to match. + base_clauses: Predicates already established (the set filter). + base_params: Their bound parameters. + falcon_response: The Falcon response to write to. + card_format: "json", "text" or "image". + face: Which face an image request wants. + version: Which image size an image request wants. + pretty: Whether to indent JSON output. + + Returns: + A card object, or a Scryfall error object. + """ + needle = fold_accents(fuzzy.strip().lower()) + words = [word for word in re.split(r"[^\w']+", needle) if word] + if not words: + return self._scryfall_respond( + falcon_response, + bad_request_error("You must provide a fuzzy or exact name parameter."), + pretty=pretty, + ) + + chosen = self._fuzzy_exact_candidate(needle, base_clauses, base_params) + if chosen is None: + candidates = self._fuzzy_containment_candidates(words, base_clauses, base_params) + if len(candidates) > 1: + return self._ambiguous(falcon_response, fuzzy, pretty=pretty) + if candidates: + chosen = candidates[0] + + if chosen is None: + chosen = self._fuzzy_similarity_candidate(needle, base_clauses, base_params) + if chosen is _AMBIGUOUS: + return self._ambiguous(falcon_response, fuzzy, pretty=pretty) + + if not chosen: + return self._scryfall_respond( + falcon_response, + not_found_error(f"No cards found matching “{fuzzy}”"), + pretty=pretty, + ) + + cards = self._cards_by_ids([str(chosen["scryfall_id"])]) + if not cards: + return self._scryfall_respond( + falcon_response, + not_found_error(f"No cards found matching “{fuzzy}”"), + pretty=pretty, + ) + return self._render_card( + cards[0], + falcon_response=falcon_response, + card_format=card_format, + face=face, + version=version, + pretty=pretty, + ) + + def _ambiguous(self, falcon_response: falcon.Response | None, name: str, *, pretty: bool) -> dict[str, Any] | None: + """Emit Scryfall's `ambiguous` error. + + Args: + falcon_response: The Falcon response to write to. + name: The name that matched more than one card. + pretty: Whether to indent JSON output. + + Returns: + The error object. + """ + return self._scryfall_respond( + falcon_response, + error_object( + code="ambiguous", + status=404, + details=f"Too many cards match ambiguous name “{name}”. Add more words to refine your search.", + ), + pretty=pretty, + ) + + def _best_printing(self, where: str, params: dict[str, Any]) -> dict[str, Any] | None: + """Return the id and name of the best-scoring printing matching a predicate. + + Args: + where: SQL predicate over `magic.cards AS card`. + params: Bound parameters. + + Returns: + A row with scryfall_id and card_name, or None. + """ + rows = self._run_query( + query=( + f"SELECT scryfall_id, card_name FROM magic.cards AS card WHERE {where} " + "ORDER BY prefer_score DESC NULLS LAST, released_at DESC LIMIT 1" + ), + params=params, + explain=False, + )["result"] + return rows[0] if rows else None + + def _fuzzy_exact_candidate( + self, + needle: str, + base_clauses: list[str], + base_params: dict[str, Any], + ) -> dict[str, Any] | None: + """Return the card whose folded name is exactly the folded query, if there is one. + + Args: + needle: The accent-folded, lowercased query. + base_clauses: Predicates already established (the set filter). + base_params: Their bound parameters. + + Returns: + The matching printing, or None. + """ + # The ENGINE first, like every other lookup on this surface. Unlike `fuzzy_card_by_name`, + # `exact_card_by_name` takes the set code, so a set filter no longer forces SQL. + row = self._engine_exact_name(needle, base_params.get("set_code")) + if row is not None: + return row + + params = {**base_params, "needle": needle} + clauses = [*base_clauses, "lower(card_name_folded) = %(needle)s"] + return self._best_printing(" AND ".join(clauses), params) + + def _fuzzy_containment_candidates( + self, + words: list[str], + base_clauses: list[str], + base_params: dict[str, Any], + ) -> list[dict[str, Any]]: + """Return one printing per distinct card name containing every query word. + + Args: + words: The folded query, split into words. + base_clauses: Predicates already established (the set filter). + base_params: Their bound parameters. + + Returns: + Up to two rows -- enough to tell "one match" from "ambiguous" without fetching more. + """ + # The ENGINE first. A LIKE per word is a sequential scan of every folded name; the engine + # narrows the same predicate through `name_trigram` -- measured 1,303 us against 11 us. + engine = self._engine_for_lookup() + if engine is not None and words: + try: + rows = engine.cards_containing_all_words( + list(words), + base_params.get("set_code"), + 2, + list(CARD_OBJECT_FIELDS), + ) + # Any engine failure falls back to SQL; it never 500s. + except Exception: + logger.exception("Engine containment match failed, falling back to SQL") + else: + # `else`, not the `try` body: a key error here is a shape mismatch, not an engine + # failure, and must not be swallowed into a silent fallback. + return [{"scryfall_id": row["scryfall_id"], "card_name": row["name"]} for row in rows] + + params = dict(base_params) + clauses = list(base_clauses) + for index, word in enumerate(words): + params[f"word_{index}"] = f"%{word}%" + clauses.append(f"lower(card_name_folded) LIKE %(word_{index})s") + return self._run_query( + query=( + "SELECT DISTINCT ON (card_name) card_name, scryfall_id " + f"FROM magic.cards AS card WHERE {' AND '.join(clauses)} " + "ORDER BY card_name, prefer_score DESC NULLS LAST LIMIT 2" + ), + params=params, + explain=False, + )["result"] + + def _fuzzy_similarity_candidate( + self, + needle: str, + base_clauses: list[str], + base_params: dict[str, Any], + ) -> dict[str, Any] | None: + """Return the typo-tolerant match, `_AMBIGUOUS` when two names are too close to separate. + + A candidate must clear FUZZY_SIMILARITY_FLOOR, and the best must lead the next distinct + card name by FUZZY_SIMILARITY_LEAD. The floor sits above pg_trgm's default 0.3 threshold, + so the index-assisted `%` prefilter is always a strict superset of what the floor admits + and no decision rests on a row the prefilter dropped. + + Args: + needle: The accent-folded, lowercased query. + base_clauses: Predicates already established (the set filter). + base_params: Their bound parameters. + + Returns: + The matching printing, `_AMBIGUOUS`, or None. + """ + # The ENGINE first, like every other lookup on this surface. `fuzzy_name_match` + # reimplements pg_trgm's similarity() exactly for this, and until now nothing called it: + # the whole of "Fuzzy Name Match and Autocomplete, Computed Not Stored" was unreachable + # from the API, which is the same defect the duplicate `_card_by_external_id` had. + # + # A set filter still goes to SQL: the engine matches on names alone and has no way to + # restrict to one set, and answering the unrestricted match would be a different card. + if not base_clauses: + engine = self._engine_for_lookup() + if engine is not None: + try: + status, row = engine.fuzzy_card_by_name( + needle, + FUZZY_SIMILARITY_FLOOR, + FUZZY_SIMILARITY_LEAD, + list(CARD_OBJECT_FIELDS), + ) + # Any engine failure falls back to SQL; it never 500s. + except Exception: + logger.exception("Engine fuzzy match failed, falling back to SQL") + else: + # The key is `scryfall_id`, which is what CARD_OBJECT_FIELDS asks for. It read + # `id` before, so every hit raised KeyError INSIDE the try above, was logged as + # an engine failure and fell through to SQL -- this fast path had never once + # returned. Reading the row in `else` is what makes the next such mismatch a + # test failure rather than a silent permanent fallback. + if status == "ambiguous": + return _AMBIGUOUS + if status == "miss": + return None + if row: + return {"scryfall_id": row["scryfall_id"], "card_name": row["name"]} + + params = {**base_params, "needle": needle, "floor": FUZZY_SIMILARITY_FLOOR} + # `%%` escapes psycopg's placeholder marker: the bare `%` operator would be read as the + # start of one. OPERATOR(magic.%) is pg_trgm's similarity match, which the folded-name GIN + # index serves. + clauses = [*base_clauses, "lower(card_name_folded) OPERATOR(magic.%%) %(needle)s"] + rows = self._run_query( + query=( + "SELECT DISTINCT ON (card_name) card_name, scryfall_id, " + "magic.similarity(lower(card_name_folded), %(needle)s) AS score " + f"FROM magic.cards AS card WHERE {' AND '.join(clauses)} " + "AND magic.similarity(lower(card_name_folded), %(needle)s) >= %(floor)s " + "ORDER BY card_name, prefer_score DESC NULLS LAST" + ), + params=params, + explain=False, + )["result"] + if not rows: + return None + ranked = sorted(rows, key=lambda row: row["score"], reverse=True) + if len(ranked) > 1 and ranked[0]["score"] - ranked[1]["score"] < FUZZY_SIMILARITY_LEAD: + return _AMBIGUOUS + return ranked[0] + + # ---------------------------------------------------------------- GET /cards/autocomplete + + @route(paths=("cards/autocomplete",)) + def scryfall_cards_autocomplete( + self, + *, + falcon_response: falcon.Response | None = None, + q: str | None = None, + pretty: str = "false", + include_extras: str = "false", # noqa: ARG002 -- declared so the 404 route listing shows it + **_: object, + ) -> dict[str, Any] | None: + """Return up to 20 card names matching a partial name. + + Args: + falcon_response: The Falcon response to write to. + q: The partial name. + pretty: Whether to indent JSON output. + include_extras: Accepted, ignored -- the corpus holds no extras to include. + + Returns: + A Catalog object of card names. + """ + is_pretty = _as_bool(pretty) + _set_cards_cache(falcon_response) + self._require_setup_complete() + needle = (q or "").strip() + min_query_length = 2 + if len(needle) < min_query_length: + return self._scryfall_respond(falcon_response, catalog_object([]), pretty=is_pretty) + # FOLDED, like `named?exact=` and the fuzzy stages above. Unfolded, an ASCII query could not + # reach a name with diacritics -- `q=eowyn` answered an empty catalog where Scryfall answers + # three Éowyn cards -- and nobody types the accent. Both paths below compare the folded name, + # so the engine and the SQL fallback keep answering alike. + needle = fold_accents(needle.lower()) + + # The ENGINE first, for the same reason the fuzzy match above now does: `autocomplete` was + # added by "Fuzzy Name Match and Autocomplete, Computed Not Stored" and nothing called it. + engine = self._engine_for_lookup() + if engine is not None: + try: + names = engine.autocomplete(needle, MAX_AUTOCOMPLETE_VALUES) + return self._scryfall_respond(falcon_response, catalog_object(list(names)), pretty=is_pretty) + # Any engine failure falls back to SQL; it never 500s. + except Exception: + logger.exception("Engine autocomplete failed, falling back to SQL") + + rows = self._run_query( + query=( + "SELECT card_name, " + "min(CASE WHEN lower(card_name_folded) LIKE %(prefix)s THEN 0 ELSE 1 END) AS rank " + "FROM magic.cards AS card WHERE lower(card_name_folded) LIKE %(needle)s " + "GROUP BY card_name ORDER BY rank, length(card_name), card_name LIMIT %(limit)s" + ), + params={"prefix": f"{needle}%", "needle": f"%{needle}%", "limit": MAX_AUTOCOMPLETE_VALUES}, + explain=False, + )["result"] + return self._scryfall_respond(falcon_response, catalog_object([row["card_name"] for row in rows]), pretty=is_pretty) + + # ---------------------------------------------------------------- GET /cards/random + + @route(paths=("cards/random",)) + def scryfall_cards_random( # noqa: PLR0913 + self, + *, + falcon_response: falcon.Response | None = None, + q: str | None = None, + format: str = "json", # noqa: A002 -- Scryfall's parameter name + face: str = "front", + version: str = DEFAULT_IMAGE_VERSION, + pretty: str = "false", + **_: object, + ) -> dict[str, Any] | None: + """Return one random card, optionally restricted by a search query. + + Args: + falcon_response: The Falcon response to write to. + q: An optional search query the card must match. + format: Response format -- json, text or image. + face: Which face an image request wants. + version: Which image size an image request wants. + pretty: Whether to indent JSON output. + + Returns: + A card object, or a Scryfall error object. + """ + is_pretty = _as_bool(pretty) + self._require_setup_complete() + where, params = "TRUE", {} + if q and q.strip(): + try: + where, params = generate_sql_query(parse_scryfall_query(q)) + except QueryBudgetExceeded as err: + # Same treatment `_search` gives the budget: the stable non-disclosing message, + # and a bounded preview in the log rather than the full query in the body. + log_ctx = bounded_query_log_context(q) + logger.info( + "Query budget exceeded (%s) preview=%r digest=%s", + err.kind, + log_ctx["query_preview"], + log_ctx["query_digest"], + ) + return self._scryfall_respond( + falcon_response, + bad_request_error(err.user_message), + pretty=is_pretty, + ) + except InvalidRegexPatternError as err: + return self._scryfall_respond( + falcon_response, + bad_request_error(err.user_message_for_query(q)), + pretty=is_pretty, + ) + except ValueError: + return self._scryfall_respond( + falcon_response, + bad_request_error(f'Failed to parse query: "{q}"'), + pretty=is_pretty, + ) + + # This response must not be cached at either layer. The HTTP cache would pin one card as + # "the" random card for the generation, and _run_query's cache would do the same a level + # down -- the draw's SQL text and parameters are identical on every call, so its first + # result would be replayed forever. Hence no-store here and an uncached draw below. + if falcon_response is not None: + falcon_response.set_header("Cache-Control", "no-store") + + # Two statements rather than ORDER BY random(): the count is deterministic, so it can go + # through the cache, and the offset scan stops as soon as it has one row where a sort would + # order the whole match set to throw all but one away. + matched = self._run_query( + query=f"SELECT count(1) AS total FROM magic.cards AS card WHERE {where}", + params=params, + explain=False, + )["result"][0]["total"] + if not matched: + return self._scryfall_respond(falcon_response, not_found_error(_NO_MATCH_DETAILS), pretty=is_pretty) + + rows = self._run_uncached( + query=( + f"SELECT {_CARD_COLUMNS} FROM magic.cards AS card WHERE {where} " + "OFFSET floor(random() * %(matched)s)::bigint LIMIT 1" + ), + params={**params, "matched": matched}, + ) + if not rows: + return self._scryfall_respond(falcon_response, not_found_error(_NO_MATCH_DETAILS), pretty=is_pretty) + return self._render_card( + to_scryfall_card(sql_row_to_engine_row(rows[0])), + falcon_response=falcon_response, + card_format=format.lower(), + face=face, + version=version, + pretty=is_pretty, + ) + + # ---------------------------------------------------------------- POST /cards/collection + + @route(paths=("cards/collection",), methods=("POST",)) + def scryfall_cards_collection( + self, + *, + falcon_response: falcon.Response | None = None, + request: falcon.Request | None = None, + pretty: str = "false", + **_: object, + ) -> dict[str, Any] | None: + """Resolve up to 75 card identifiers in one request. + + Args: + falcon_response: The Falcon response to write to. + request: The Falcon request, whose JSON body carries the identifiers. + pretty: Whether to indent JSON output. + + Returns: + A List object whose `data` holds the cards found and whose `not_found` holds the + identifiers that resolved to nothing, or a Scryfall error object. + """ + is_pretty = _as_bool(pretty) + # A shared cache keys on the URL and this route's answer depends entirely on the BODY, + # so it is private and always revalidated -- api.scryfall.com sends the same. + if falcon_response is not None: + falcon_response.set_header("Cache-Control", "max-age=0, private, must-revalidate") + self._require_setup_complete() + try: + body = request.get_media() if request is not None else None + except (falcon.MediaMalformedError, falcon.MediaNotFoundError): + body = None + if not isinstance(body, dict) or not isinstance(body.get("identifiers"), list): + return self._scryfall_respond( + falcon_response, + error_object( + code="validation_error", + status=422, + details="The request body must be a JSON object with an `identifiers` array.", + ), + pretty=is_pretty, + ) + + identifiers = body["identifiers"] + if len(identifiers) > MAX_COLLECTION_IDENTIFIERS: + return self._scryfall_respond( + falcon_response, + error_object( + code="validation_error", + status=422, + details=f"A maximum of {MAX_COLLECTION_IDENTIFIERS} card references may be submitted at once.", + ), + pretty=is_pretty, + ) + + found: list[dict[str, Any]] = [] + not_found: list[dict[str, Any]] = [] + seen: set[str] = set() + for identifier in identifiers: + card = self._resolve_identifier(identifier) if isinstance(identifier, dict) else None + if card is None: + not_found.append(identifier) + continue + if card["id"] in seen: + continue + seen.add(card["id"]) + found.append(card) + + return self._scryfall_respond(falcon_response, card_list(found, not_found=not_found), pretty=is_pretty) + + def _resolve_identifier(self, identifier: dict[str, Any]) -> dict[str, Any] | None: + """Resolve one collection identifier to a card. + + Args: + identifier: One entry of the request's `identifiers` array. + + Returns: + The card it names, or None when nothing matched or the shape is not one Scryfall + defines. + """ + if "id" in identifier and _is_uuid(str(identifier["id"])): + return self._card_by_scryfall_id(str(identifier["id"])) + if "oracle_id" in identifier and _is_uuid(str(identifier["oracle_id"])): + return self._card_by_oracle_id(str(identifier["oracle_id"])) + if "illustration_id" in identifier and _is_uuid(str(identifier["illustration_id"])): + return self._card_by_illustration_id(str(identifier["illustration_id"])) + if "mtgo_id" in identifier: + return self._card_by_external_id("mtgo", _as_int(str(identifier["mtgo_id"]))) + if "multiverse_id" in identifier: + return self._card_by_multiverse_id(_as_int(str(identifier["multiverse_id"]))) + if "set" in identifier and "collector_number" in identifier: + return self._fetch_one_card( + "lower(card_set_code) = lower(%(set_code)s) AND collector_number = %(number)s", + {"set_code": str(identifier["set"]), "number": str(identifier["collector_number"])}, + ) + if "name" in identifier: + set_code = identifier.get("set") + return self._card_by_name_identifier( + str(identifier["name"]), + str(set_code) if set_code else None, + ) + return None + + def _card_by_name_identifier(self, name: str, set_code: str | None) -> dict[str, Any] | None: + """Resolve a collection identifier's `name` -- a NAME LOOKUP with its own keys. + + Not `named?exact=`'s keys, which is why this is its own engine entry point and not a call + to the one beside it: `{"name":"Delver of Secrets // Insectile Aberration"}` is not_found on + api.scryfall.com while `exact=` of that same string answers the card. The block above + `_collate_name` carries the measurements for both. + + Four things the SQL this replaces got wrong, each measured: it never looked at the BACK face + (`{"name":"Insectile Aberration"}` answers the card there and missed here), it accepted the + joined name (`{"name":"Fire // Ice"}` is not_found there and answered here), it read a + five-part name as having a front face (`{"name":"Who"}` is not_found there and answered + und/75 here), and it compared `card_name` as posted -- so no accent, punctuation or spacing + difference resolved, and `{"name":" Lightning Bolt "}` missed on its own whitespace. + + The ENGINE first, through `_engine_card`, like every other identifier this route accepts: + a genuine "no such card" from a loaded store IS the answer, and only a store that cannot + serve falls through to SQL. `_engine_exact_name` conflates those two, which is right for + `named?exact=` -- there the SQL is a second chance at the same question -- and wrong here, + where SQL answering a needle the engine reported not_found would be the pre-existing bug + coming back through the fallback. + + Args: + name: The identifier's `name` value, as posted. + set_code: The identifier's `set`, which FILTERS the lookup -- `{"name":"Delver of + Secrets","set":"mid"}` answers mid/47, the same card in the set asked for, and a + card with no printing in that set drops out rather than answering another printing. + + Returns: + The card it names, or None for not_found. + """ + collated = _collate_name(name) + # A needle with no alphanumeric character is nobody's name. Answered here rather than as a + # query, because `%(collated)s = ''` would match any card whose name is punctuation alone. + if not collated: + return None + found = self._engine_card( + lambda e: e.collection_card_by_name(fold_accents(name.strip().lower()), set_code, list(CARD_OBJECT_FIELDS)), + ) + if found is not _ENGINE_MISS: + return found + clauses = [_COLLECTION_NAME_MATCH] + params: dict[str, Any] = {"collated": collated} + if set_code: + clauses.append("lower(card_set_code) = lower(%(set_code)s)") + params["set_code"] = set_code + return self._fetch_one_card(" AND ".join(clauses), params, rank_first=_WHOLE_NAME_FIRST) + + # ---------------------------------------------------------------- GET /cards and /cards/... + + @route() + def cards( # noqa: PLR0913 + self, + identifier: str = "", + number: str = "", + suffix: str = "", + *, + falcon_response: falcon.Response | None = None, + request: falcon.Request | None = None, + request_host: str = "", + page: str = "1", + format: str = "json", # noqa: A002 -- Scryfall's parameter name + face: str = "front", + version: str = DEFAULT_IMAGE_VERSION, + pretty: str = "false", + **_: object, + ) -> dict[str, Any] | None: + """Serve every `/cards/*` route the five named sub-routes do not claim. + + The path shapes, by segment count: + + - `/cards` -- every card, paginated. + - `/cards/:id` -- one card by Scryfall id. + - `/cards/:namespace/:id` -- one card by multiverse, MTGO, Arena, TCGplayer or Cardmarket id. + - `/cards/:id/rulings` -- the rulings for one card. + - `/cards/:code/:number` -- one card by set code and collector number. + - `/cards/:code/:number/:lang` -- the same, in one language. + - `/cards/:namespace/:id/rulings` and `/cards/:code/:number/rulings` -- rulings, addressed + the same two ways. + + Args: + identifier: First path segment: a Scryfall id, an external id namespace, or a set code. + number: Second path segment: an external id, a collector number, or "rulings". + suffix: Third path segment: a language code or "rulings". + falcon_response: The Falcon response to write to. + request: The Falcon request, read for the scheme `next_page` should use. + request_host: Host the request arrived on, used to build `next_page`. + page: 1-based page number, for the unfiltered `/cards` listing. + format: Response format -- json, text or image. + face: Which face an image request wants. + version: Which image size an image request wants. + pretty: Whether to indent JSON output. + + Returns: + A card, List or Catalog object, or a Scryfall error object. + """ + is_pretty = _as_bool(pretty) + _set_cards_cache(falcon_response) + self._require_setup_complete() + + if not identifier: + return self._all_cards_page( + falcon_response=falcon_response, + request=request, + request_host=request_host, + page=page, + pretty=is_pretty, + ) + + wants_rulings = "rulings" in (number, suffix) + card = self._resolve_path_card(identifier, number, suffix, wants_rulings=wants_rulings) + if card is None: + return self._scryfall_respond( + falcon_response, + not_found_error(_miss_details(identifier, number, suffix)), + pretty=is_pretty, + ) + if wants_rulings: + return self._scryfall_respond(falcon_response, self._rulings_for(card), pretty=is_pretty) + return self._render_card( + card, falcon_response=falcon_response, card_format=format.lower(), face=face, version=version, pretty=is_pretty + ) + + def _resolve_path_card(self, identifier: str, number: str, suffix: str, *, wants_rulings: bool) -> dict[str, Any] | None: + """Resolve the card a `/cards/...` path addresses. + + Args: + identifier: First path segment. + number: Second path segment. + suffix: Third path segment. + wants_rulings: Whether a trailing "rulings" segment was consumed from the path. + + Returns: + The card, or None when the path addresses nothing. + """ + # Drop the trailing "rulings" so the rest reads as a plain card address. + if wants_rulings: + if suffix == "rulings": + suffix = "" + else: + number, suffix = "", "" + + if identifier in _EXTERNAL_ID_NAMESPACES: + external_id = _as_int(number) + if external_id is None: + return None + if identifier == "multiverse": + return self._card_by_multiverse_id(external_id) + return self._card_by_external_id(identifier, external_id) + + if not number: + if not _is_uuid(identifier): + return None + return self._card_by_scryfall_id(identifier) + + clauses = ["lower(card_set_code) = lower(%(set_code)s)", "collector_number = %(number)s"] + params: dict[str, Any] = {"set_code": identifier, "number": number} + # Scryfall defaults the language segment to English rather than to "any language". + clauses.append("raw_card_blob ->> 'lang' = %(lang)s") + params["lang"] = suffix or "en" + return self._fetch_one_card(" AND ".join(clauses), params) + + def _card_by_multiverse_id(self, multiverse_id: int | None) -> dict[str, Any] | None: + """Fetch a card by Gatherer multiverse id. + + Args: + multiverse_id: The id to match. + + Returns: + The card, or None when nothing matched. + """ + if multiverse_id is None: + return None + return self._fetch_one_card( + "raw_card_blob -> 'multiverse_ids' @> %(value)s::jsonb", + {"value": str(multiverse_id)}, + ) + + def _all_cards_page( + self, + *, + falcon_response: falcon.Response | None, + request: falcon.Request | None, + request_host: str, + page: str, + pretty: bool, + ) -> dict[str, Any] | None: + """Serve one page of the unfiltered `/cards` listing. + + Args: + falcon_response: The Falcon response to write to. + request: The Falcon request, read for the scheme `next_page` should use. + request_host: Host the request arrived on. + page: 1-based page number. + pretty: Whether to indent JSON output. + + Returns: + A List object of cards, or a Scryfall error object. + """ + # `or 1` would swallow page=0 into page=1; an unparseable page defaults, a non-positive + # one is rejected below. + parsed_page = _as_int(page) + page_number = 1 if parsed_page is None else parsed_page + if page_number < 1: + return self._scryfall_respond( + falcon_response, + bad_request_error("The page parameter must be a positive integer."), + pretty=pretty, + ) + total = self._run_query(query="SELECT count(1) AS total FROM magic.cards", explain=False)["result"][0]["total"] + rows = self._run_query( + query=( + f"SELECT {_CARD_COLUMNS} FROM magic.cards AS card " + "ORDER BY card_name, card_set_code, collector_number_int, collector_number " + "LIMIT %(limit)s OFFSET %(offset)s" + ), + params={"limit": PAGE_SIZE, "offset": (page_number - 1) * PAGE_SIZE}, + explain=False, + )["result"] + if not rows: + return self._scryfall_respond(falcon_response, not_found_error(_NO_MATCH_DETAILS), pretty=pretty) + + cards = [to_scryfall_card(row) for row in rows] + has_more = (page_number - 1) * PAGE_SIZE + len(cards) < total + next_page = None + if has_more: + next_page = objects.build_page_url(_self_base_url(request, request_host, "/cards"), {}, page_number + 1) + return self._scryfall_respond( + falcon_response, + card_list(cards, total_cards=total, has_more=has_more, next_page=next_page), + pretty=pretty, + ) + + def _rulings_for(self, card: dict[str, Any]) -> dict[str, Any]: + """Build the rulings List object for a card. + + Newest first, which is the order api.scryfall.com serves and NOT the ascending one this + started with. Measured on 2026-08-12 over the cards whose rulings span more than one date: + 16 of 16 came back `published_at` descending, 0 ascending -- so ascending inverted every + multi-date card for a client that had changed nothing but its base URL. Three concrete + examples, as Scryfall returns them: Kindred Discovery 2023-09-01, 2022-06-10, 2022-06-10, + 2017-08-25; Eye of the Storm 2006-02-01, 2006-01-01, 2005-10-01 x3; Diabolic Intent + 2022-10-14 x2, 2013-04-15 x2, 2004-10-04. + + WITHIN one date the order cannot be reproduced from the bulk file, and `comment` is a + deterministic stand-in rather than a claim to match. Scryfall orders same-date rulings by an + internal ruling id; the file carries no id, and none of the file's own order, that order + reversed, comment ascending or comment descending matched on any of 10 sampled cards that + have a date carrying several rulings. That is most cards -- 13,847 of the 19,770 with + rulings, against the 2026-08-11 dump -- so the remaining 5,923 (one ruling, or one per date) + are the ones this now matches exactly. See docs/issues/local-scryfall-cards-api.md. + + Args: + card: The card whose oracle id the rulings hang off. + + Returns: + A List object of Ruling objects, empty when the card has none. + """ + oracle_id = card.get("oracle_id") + if not oracle_id: + return card_list([]) + rows = self._run_query( + query=( + "SELECT oracle_id, source, published_at, comment FROM magic.rulings " + "WHERE oracle_id = %(oracle_id)s ORDER BY published_at DESC, comment" + ), + params={"oracle_id": str(oracle_id)}, + explain=False, + )["result"] + return card_list([ruling_object(row) for row in rows]) diff --git a/api/scryfall_reference_import.py b/api/scryfall_reference_import.py new file mode 100644 index 000000000..8d6e0e0c6 --- /dev/null +++ b/api/scryfall_reference_import.py @@ -0,0 +1,165 @@ +"""Mirror Scryfall's reference data — sets, catalogs and card symbols — into `magic`. + +These three are not bulk data. Scryfall publishes them as ordinary API endpoints, small enough to +fetch whole: 1,047 sets, twenty catalogs totalling around 60,000 strings, and 84 card symbols. So +this module talks to `api.scryfall.com` directly through the bulk fetcher's retrying session rather +than through `stream_data_for_key`. + +Every load is a whole-table replace inside one transaction, for the reason the rulings load is: the +upstream response is the entire truth each time, a set can be renamed or withdrawn before release, +and a replace cannot get the pruning wrong. `DELETE` rather than `TRUNCATE` so readers keep seeing +the previous contents through MVCC instead of blocking on an ACCESS EXCLUSIVE lock. + +Why mirrored and not derived: the corpus cannot answer these. A Set object carries eight fields no +card carries, `card_count` counts printings this instance deliberately never imported, and a card +symbol's `svg_uri` exists nowhere in the card data. The full argument is in the migration header, +`api/db/2026-08-11-02-scryfall-sets-catalogs-symbology.sql`. +""" + +from __future__ import annotations + +import logging +import time +from typing import TYPE_CHECKING, Any + +from psycopg.types.json import Jsonb + +from api.scryfall_compat.reference_routes import CATALOG_NAMES + +if TYPE_CHECKING: + import psycopg_pool + + from api.scryfall_bulk_data_fetcher import ScryfallBulkDataFetcher + +logger = logging.getLogger(__name__) + + +def import_sets(conn_pool: psycopg_pool.ConnectionPool, fetcher: ScryfallBulkDataFetcher) -> dict[str, Any]: + """Replace `magic.sets` with Scryfall's current set list. + + Args: + conn_pool: Pool to run the load through. + fetcher: Fetcher whose session the request goes through. + + Returns: + A summary of the load. + """ + start = time.monotonic() + payload = fetcher.fetch_api_json("sets") + sets = [entry for entry in payload.get("data", []) if entry.get("id") and entry.get("code")] + + rows = [ + { + "id": entry["id"], + "code": entry["code"], + "tcgplayer_id": entry.get("tcgplayer_id"), + "position": position, + "set_object": Jsonb(entry), + } + for position, entry in enumerate(sets) + ] + + with conn_pool.connection() as conn: + with conn.cursor() as cursor: + cursor.execute("DELETE FROM magic.sets") + cursor.executemany( + "INSERT INTO magic.sets (id, code, tcgplayer_id, position, set_object) " + "VALUES (%(id)s, %(code)s, %(tcgplayer_id)s, %(position)s, %(set_object)s) " + "ON CONFLICT (id) DO NOTHING", + rows, + ) + conn.commit() + + result = { + "duration_seconds": round(time.monotonic() - start, 2), + "sets_imported": len(rows), + "sets_with_tcgplayer_id": sum(1 for row in rows if row["tcgplayer_id"] is not None), + } + logger.info("Set import complete: %s", result) + return result + + +def import_catalogs(conn_pool: psycopg_pool.ConnectionPool, fetcher: ScryfallBulkDataFetcher) -> dict[str, Any]: + """Replace `magic.catalogs` with the current contents of every documented catalog. + + A catalog that fails to fetch is skipped rather than fatal, and its previous row is left in + place: nineteen fresh catalogs and one stale one is a better answer than aborting the refresh. + + Args: + conn_pool: Pool to run the load through. + fetcher: Fetcher whose session the requests go through. + + Returns: + A summary of the load. + """ + start = time.monotonic() + fetched: dict[str, list[str]] = {} + failed: list[str] = [] + for name in CATALOG_NAMES: + try: + payload = fetcher.fetch_api_json(f"catalog/{name}") + except Exception: + logger.exception("Catalog %s could not be fetched; keeping the previous contents", name) + failed.append(name) + continue + values = payload.get("data") + if isinstance(values, list): + fetched[name] = [value for value in values if isinstance(value, str)] + else: + failed.append(name) + + with conn_pool.connection() as conn: + with conn.cursor() as cursor: + # Only the catalogs that came back are replaced, so a failed fetch keeps its old row. + cursor.executemany( + "INSERT INTO magic.catalogs (name, entries) VALUES (%(name)s, %(entries)s) " + "ON CONFLICT (name) DO UPDATE SET entries = EXCLUDED.entries", + [{"name": name, "entries": Jsonb(values)} for name, values in fetched.items()], + ) + conn.commit() + + result = { + "duration_seconds": round(time.monotonic() - start, 2), + "catalogs_imported": len(fetched), + "catalogs_failed": len(failed), + "values_imported": sum(len(values) for values in fetched.values()), + } + logger.info("Catalog import complete: %s", result) + return result + + +def import_symbology(conn_pool: psycopg_pool.ConnectionPool, fetcher: ScryfallBulkDataFetcher) -> dict[str, Any]: + """Replace `magic.card_symbols` with Scryfall's current symbol list. + + Args: + conn_pool: Pool to run the load through. + fetcher: Fetcher whose session the request goes through. + + Returns: + A summary of the load. + """ + start = time.monotonic() + payload = fetcher.fetch_api_json("symbology") + symbols = [entry for entry in payload.get("data", []) if entry.get("symbol")] + + rows = [ + {"symbol": entry["symbol"], "position": position, "symbol_object": Jsonb(entry)} for position, entry in enumerate(symbols) + ] + + with conn_pool.connection() as conn: + with conn.cursor() as cursor: + cursor.execute("DELETE FROM magic.card_symbols") + cursor.executemany( + "INSERT INTO magic.card_symbols (symbol, position, symbol_object) " + "VALUES (%(symbol)s, %(position)s, %(symbol_object)s) " + "ON CONFLICT (symbol) DO NOTHING", + rows, + ) + conn.commit() + + result = { + "duration_seconds": round(time.monotonic() - start, 2), + "symbols_imported": len(rows), + } + logger.info("Symbology import complete: %s", result) + return result diff --git a/api/static/app.js b/api/static/app.js index cb0050c32..046b3e28e 100644 --- a/api/static/app.js +++ b/api/static/app.js @@ -105,6 +105,7 @@ class CardSearch { this.currentRequestUrl = null; // URL of the in-flight request, if any this.imageObserver = null; this.cardsData = new Map(); // Store card data by ID + this.backFaceKeys = new Map(); // "set/collector" -> bool, cached back-face image probes this.lastCompletedUrl = null; // URL whose results are currently displayed; null when results are cleared this.isAscending = true; // Track order direction this.currentCardCount = 0; // Track current number of cards displayed for resize handling @@ -962,6 +963,9 @@ class CardSearch { .join(''); } + // Runs against the DOM, so it enhances SSR-rendered and JS-rendered cards alike. + this.enhanceDoubleFacedCards(); + // Record arrival time; we only push this state when leaving if they stayed > DWELL_MS and it's not already saved (updateURL) const url = this.buildCurrentSearchUrl(); window.history.replaceState({ arrivalTime: Date.now() }, '', url); @@ -1008,9 +1012,134 @@ class CardSearch { this.resultsContainer.style.gridTemplateColumns = `repeat(${actualColumns}, 1fr)`; } - buildImageUrl(card, size) { - const face = card.face_idx || 1; - return `https://d1hot9ps2xugbc.cloudfront.net/img/${card.set_code}/${card.collector_number}/${face}/${size}.webp`; + buildImageUrl(card, size, face) { + const resolvedFace = face || card.face_idx || 1; + return `https://d1hot9ps2xugbc.cloudfront.net/img/${card.set_code}/${card.collector_number}/${resolvedFace}/${size}.webp`; + } + + buildSrcset(card, face) { + return ['280', '388', '538', '745'].map(size => `${this.buildImageUrl(card, size, face)} ${size}w`).join(', '); + } + + // ── Double-faced cards: flip button (progressive enhancement) ── + // The rendered card HTML is shared with the no-JS server renderer (parity fixture), so the + // flip button is injected AFTER render rather than templated in. A card gets one when a + // face-2 image exists on the CDN — which is exactly the set of cards with a physical back + // face (transform/MDFC), since the image sync only uploads face 2 for those. Split and + // adventure cards share a "//" name but have no face-2 image, so they correctly get none. + + cardBackFaceKey(card) { + return `${card.set_code}/${card.collector_number}`; + } + + probeBackFace(card, onExists) { + if (!card.name || !card.name.includes(' // ') || !card.set_code || !card.collector_number) { + return; + } + const key = this.cardBackFaceKey(card); + if (this.backFaceKeys.has(key)) { + if (this.backFaceKeys.get(key)) onExists(); + return; + } + const probe = new Image(); + probe.onload = () => { + this.backFaceKeys.set(key, true); + onExists(); + }; + probe.onerror = () => this.backFaceKeys.set(key, false); + probe.src = this.buildImageUrl(card, '280', 2); + } + + enhanceDoubleFacedCards() { + for (const [cardId, card] of this.cardsData) { + this.probeBackFace(card, () => this.attachGridFlipButton(cardId, card)); + } + } + + // The button is positioned as a percentage of the CARD IMAGE, so it needs a + // containing block that is exactly the image's box. Neither natural parent is: + // a grid tile is the image plus the name/type/text rows, and the modal's image + // wrapper is a flex area far wider than the picture inside it — which is how + // the button ended up floating in the margin beside a large card. + // + // The frame cannot be the that already wraps the image: a