From 11f2075551ae5099cba207a942c46233a2a2fe5a Mon Sep 17 00:00:00 2001 From: daveycodez Date: Tue, 11 Aug 2026 02:11:11 -0700 Subject: [PATCH 1/2] Import: Skip the Memorabilia Sets Scryfall Hides, and Add is:oversized MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Scryfall hides `set_type: memorabilia` printings — World Championship decks, Collectors' Edition, 30th Anniversary, the oversized promos, 99 sets in all — from any search that does not name their set. Measured against api.scryfall.com on 2026-08-11: !"Ancestral Recall" 9 printings returned, 18 exist !"Birds of Paradise" 42 printings returned, 43 exist set:cei returns the hidden printing -set:lea returns all 17 others, hidden ones included border:black / year<=1994 / r:rare / cn:48 / st:core / is:promo no unhiding: only the 9 visible printings Importing them makes ordinary queries disagree. Concretely they supply the CHEAPEST printing for 184 cards, which is exactly the printing a price ordering is defined to return — `order=usd` was ranking Birds of Paradise by a $4.98 World Championship copy where Scryfall ranks it by a $9.60 one. Dropped at import, beside the `funny` exclusion it mirrors, rather than filtered at query time. That is the load-bearing decision and it was made on measurement, having built the query-time version first: To be correct a query-time predicate has to sit in the FILTER TREE. The representative walk picks the `prefer` best among printings that pass the residual, so a flag outside the tree lets a hidden printing stand for its card — and `total` is derived five different ways across the plans, so a predicate the filter does not carry has to be re-implemented in each. But a conjunct present on every query breaks four of the six physical plans. `PlanePopcountOrder` and `all_match_known` gate on the residual being literally `FilterExpr::True`; `CardRangePopcount` and `PrintingRangeScan` gate on the range being BARE. None can be taught to look through it: the two range fastpaths emit straight from a value-index slice and report the slice WIDTH as their total, so looking through would emit hidden printings and miscount them in one go. Measured at +59..115us per query on a 112,932-printing store, with `PlanePopcountOrder` — the plan serving the default `unique=card` plane shape — lost on every query. What this gives up: `set:cei` and its 98 siblings return nothing, where Scryfall returns them. No CARD is lost — measured, 0 of 31,724 cards are printed only in memorabilia sets — so this changes which printing represents a card, never whether the card is findable. `is:oversized` joins BOOLEAN_IS_TAGS alongside it. It is one of the filters that unhides on Scryfall, and it stays meaningful after this exclusion: of 726 oversized printings, only 253 are memorabilia — the rest are planechase, commander, archenemy, vanguard and promo, which this does not touch. Low cardinality, so it does not carry the foil/promo/reprint memory concern the existing comment records. The filter test pins the LITERAL set_type rather than the constant: driving both sides off the same constant made it self-referential and it passed with the constant set to anything at all, caught by flipping it. --- api/admin_resource.py | 5 +++++ api/card_processing.py | 25 +++++++++++++++++++++++++ api/tests/test_card_processing.py | 23 ++++++++++++++++++++++- 3 files changed, 52 insertions(+), 1 deletion(-) diff --git a/api/admin_resource.py b/api/admin_resource.py index 3efc420f1..43f736dc4 100644 --- a/api/admin_resource.py +++ b/api/admin_resource.py @@ -125,6 +125,11 @@ "league": "cards.raw_card_blob->'promo_types' @> '\"league\"'", "masterpiece": "cards.raw_card_blob->>'set_type' = 'masterpiece'", "media_insert": "cards.raw_card_blob->'promo_types' @> '\"mediainsert\"'", + # Low cardinality -- memorabilia and the oversized promo sets, ~2.7% of printings -- so it + # does not carry the foil/promo/reprint concern in the header above. It is also one of the + # filters that turns OFF default visibility (see _unhides_extras), which is only expressible + # at all once the tag exists. + "oversized": "cards.raw_card_blob->'oversized' = 'true'::jsonb", # "Partner with " cards carry a plain "Partner" keyword alongside it (verified # against the corpus), so checking for "Partner" alone already covers both. "partner": "cards.raw_card_blob->'keywords' @> '\"Partner\"'", diff --git a/api/card_processing.py b/api/card_processing.py index 3de650c86..851c68409 100644 --- a/api/card_processing.py +++ b/api/card_processing.py @@ -20,6 +20,11 @@ # for any card with no type in this set. Title-cased to match parse_type_line(). PERMANENT_CARD_TYPES = {"Artifact", "Battle", "Creature", "Enchantment", "Land", "Planeswalker"} +# Scryfall's set_type for the products that are collectible objects rather than tournament-legal +# printings: World Championship decks, Collectors' Edition, 30th Anniversary, the oversized promo +# sets -- 30a, ced, cei, ovnt, ptc, o90p, olep, wc97..wc04, 99 sets in all. +MEMORABILIA_SET_TYPE = "memorabilia" + def parse_type_line(type_line: str) -> tuple[list[str], list[str]]: """Parse the type line of a card.""" @@ -127,6 +132,26 @@ def preprocess_card(card: dict[str, Any]) -> list[dict[str, Any]]: # noqa: PLR0 return [] if card.get("set_type") == "funny": return [] + # Memorabilia, for the same reason one line up: a product that is a collectible object rather + # than a tournament-legal printing. Scryfall hides these from any search that does not name + # their set -- measured 2026-08-11, `!"Ancestral Recall"` returns 9 of its 18 printings and + # `!"Birds of Paradise"` 42 of 43 -- so importing them makes ordinary queries disagree with it. + # Concretely they supplied the CHEAPEST printing for 184 cards, which is exactly the printing a + # price ordering is defined to return. + # + # Dropped at import rather than filtered at query time, and that is the load-bearing decision. + # To be correct a query-time predicate has to sit in the filter TREE -- the representative walk + # picks among printings that pass the residual, so a flag outside it would let a hidden + # printing stand for its card -- and a conjunct present on every query breaks four of the six + # physical plans, which gate on the filter being literally `True` or on a range being bare: + # PlanePopcountOrder, CardRangePopcount, PrintingRangeScan, and `all_match_known`'s + # constant-count arms. Measured at +59..115us per query on a 112,932-printing store. + # + # What this gives up instead: `set:cei` and its 98 siblings return nothing, where Scryfall + # returns them. No CARD is lost -- measured, 0 of 31,724 cards are printed only in memorabilia + # sets -- so it changes which printing represents a card, never whether the card is findable. + if card.get("set_type") == MEMORABILIA_SET_TYPE: + return [] # Filter out unplayable cards: Cards and Tokens type_line = card.get("type_line") diff --git a/api/tests/test_card_processing.py b/api/tests/test_card_processing.py index e28ee6af4..32038d022 100644 --- a/api/tests/test_card_processing.py +++ b/api/tests/test_card_processing.py @@ -7,7 +7,7 @@ import uuid from typing import Any -from api.card_processing import extract_frame_data_from_raw_card, preprocess_card +from api.card_processing import MEMORABILIA_SET_TYPE, extract_frame_data_from_raw_card, preprocess_card # Project root directory for accessing sample data _PROJECT_ROOT = pathlib.Path(__file__).parent.parent.parent @@ -209,6 +209,27 @@ def test_preprocess_card_filters_funny_sets(self) -> None: result = preprocess_card(invalid_card) assert result == [] + def test_preprocess_card_filters_memorabilia_sets(self) -> None: + """Memorabilia printings are not imported. + + Scryfall hides them from any search that does not name their set, so importing them makes + ordinary queries disagree with it. Measured 2026-08-11: `!"Ancestral Recall"` returns 9 of its 18 printings on Scryfall, and + the 9 it omits are exactly the memorabilia ones (30a, ced, cei, ovnt) plus a digital set. + """ + # The LITERAL, not MEMORABILIA_SET_TYPE: driving both sides off the same constant makes the + # test self-referential, and it then passes with the constant set to anything at all. + assert preprocess_card(create_test_card(set_type="memorabilia")) == [] + # Pinned separately, so the constant is still what names the Scryfall set_type. + assert MEMORABILIA_SET_TYPE == "memorabilia" + + def test_preprocess_card_keeps_ordinary_sets(self) -> None: + """The exclusion is on set_type alone — a normal expansion is untouched. + + Paired with the test above so a predicate that accidentally dropped everything (an `in` + against the wrong operand, say) fails here rather than looking like a working filter. + """ + assert len(preprocess_card(create_test_card(set_type="expansion"))) == 1 + def test_preprocess_card_filters_card_type(self) -> None: """Test preprocess_card filters out cards with Card type.""" invalid_card = create_test_card( From 2982cb0e09481ab883c2b40f828f7736aab64592 Mon Sep 17 00:00:00 2001 From: daveycodez Date: Thu, 27 Aug 2026 21:12:27 -0700 Subject: [PATCH 2/2] Import: Also Skip Oversized Printings, and Drop the Now-Empty is:oversized Tag Per the measurement in the PR discussion (2026-08-27 against api.scryfall.com): every card that exists only oversized is not_legal in every format and already refused by the legality gate, so excluding oversized at import removes exactly 10 printings past the memorabilia exclusion -- the p09/p10/p11 oversized box-topper promos, each of which has normal-sized printings. With oversized never imported the is:oversized tag would be permanently empty, so it goes too (its BOOLEAN_IS_TAGS entry was dropped in the merge with main). --- api/card_processing.py | 8 ++++++++ api/tests/test_card_processing.py | 14 ++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/api/card_processing.py b/api/card_processing.py index 851c68409..843612582 100644 --- a/api/card_processing.py +++ b/api/card_processing.py @@ -152,6 +152,14 @@ def preprocess_card(card: dict[str, Any]) -> list[dict[str, Any]]: # noqa: PLR0 # sets -- so it changes which printing represents a card, never whether the card is findable. if card.get("set_type") == MEMORABILIA_SET_TYPE: return [] + # Oversized printings, same reasoning. Every card that exists ONLY oversized -- all 207 + # planes, all 102 schemes, all 32 paper Vanguard avatars, Garruk the Slayer -- is not_legal + # in every format, so the legality gate above already refuses it before this line runs. + # Measured 2026-08-27: after the gates above, exactly 240 oversized printings survive, 230 of + # them memorabilia; this line removes the last 10, the p09/p10/p11 oversized box-topper + # promos, each of which has normal-sized printings. So no card is lost here either. + if card.get("oversized"): + return [] # Filter out unplayable cards: Cards and Tokens type_line = card.get("type_line") diff --git a/api/tests/test_card_processing.py b/api/tests/test_card_processing.py index 32038d022..b2205a781 100644 --- a/api/tests/test_card_processing.py +++ b/api/tests/test_card_processing.py @@ -230,6 +230,20 @@ def test_preprocess_card_keeps_ordinary_sets(self) -> None: """ assert len(preprocess_card(create_test_card(set_type="expansion"))) == 1 + def test_preprocess_card_filters_oversized_printings(self) -> None: + """Oversized printings are not imported. + + Every card that exists ONLY oversized (planes, schemes, Vanguard avatars) is not_legal in + every format and already refused by the legality gate, so this test card carries legal + legalities to reach the oversized check itself. Measured 2026-08-27: past the earlier gates + this drops exactly 10 printings, the p09/p10/p11 oversized box-topper promos, each of which + has normal-sized printings. + + `test_preprocess_card_keeps_ordinary_sets` above is the paired keep-side: its card carries + no `oversized` field at all, matching the bulk objects where the flag is false. + """ + assert preprocess_card(create_test_card(set_type="expansion", oversized=True)) == [] + def test_preprocess_card_filters_card_type(self) -> None: """Test preprocess_card filters out cards with Card type.""" invalid_card = create_test_card(