From 0b985cd9c6f2d0244d2ec07b287e3b6bea6d747a Mon Sep 17 00:00:00 2001 From: daveycodez Date: Sat, 22 Aug 2026 14:00:09 -0400 Subject: [PATCH 1/3] Parsing: Normalize Typographic (Smart) Quotes in Query Strings Every word processor and phone keyboard turns a typed apostrophe into U+2019 and typed double quotes into U+201C/U+201D, so a pasted query carries them constantly. This parser read them as ordinary letters, which made a query for Gaea's Blessing (curly apostrophe) a search for a name containing that curly apostrophe: no rows, no error, no clue. fold_typographic_quotes folds exactly four characters -- U+2018/U+2019 to the ASCII apostrophe, U+201C/U+201D to the ASCII double quote -- and leaves everything else quotation-shaped alone. Measured against api.scryfall.com by putting each candidate around a phrase and asking whether it searched as one term: the guillemets, low-9 quotes, primes, fullwidth forms, CJK brackets, ornate quotes, backtick, acute and U+02BC all stay literal and match nothing, so widening the table would silently answer more than Scryfall does. It is a character substitution over the whole query, not a rule about quoted regions -- a curly apostrophe inside straight double quotes folds too, which is the only reason `name:"Gaea's Blessing"` (with a curly apostrophe) finds the card at all. Lives in spans.py, the leaf module already shared by both parsers and the typeahead balancer for exactly this kind of character-level rule, and applies before anything reads a character as a delimiter: at the top of parse_query, parse_search_query, and balance_partial_query. The balancer and the lexer have to agree about what a quote is, or a typed opening curly quote balances to nothing and then fails to lex as an unclosed `name:'` once parse_query folds it. Both parsers take the change, so test_parser_parity keeps them from diverging. Closes #970. --- api/parsing/hand_parser.py | 6 +- api/parsing/parsing_f.py | 7 +- api/parsing/pyparsing_based.py | 8 +- api/parsing/spans.py | 32 +++++++ api/parsing/tests/test_typographic_quotes.py | 97 ++++++++++++++++++++ 5 files changed, 147 insertions(+), 3 deletions(-) create mode 100644 api/parsing/tests/test_typographic_quotes.py diff --git a/api/parsing/hand_parser.py b/api/parsing/hand_parser.py index 0682de0e9..ec3ba872c 100644 --- a/api/parsing/hand_parser.py +++ b/api/parsing/hand_parser.py @@ -29,7 +29,7 @@ TrueNode, flatten_nested_operations, ) -from api.parsing.spans import QUOTE_CHARS, brace_close_index, find_close_index, unescape +from api.parsing.spans import QUOTE_CHARS, brace_close_index, find_close_index, fold_typographic_quotes, unescape # ── Alias → parser-class lookup ────────────────────────────────────────────── @@ -816,6 +816,10 @@ def parse_query(src: str | None) -> Query: """ if not src or not src.strip(): return Query(TrueNode()) + # Before the lexer, because a curly quote has to BE a quote by the time a term boundary is + # decided; and rebinding `src` so the "Failed to lex/parse" messages echo the query the parser + # actually read rather than the one the user pasted. + src = fold_typographic_quotes(src) try: tokens = tokenize(src) except LexError as exc: diff --git a/api/parsing/parsing_f.py b/api/parsing/parsing_f.py index e75588712..f69b1d7b3 100644 --- a/api/parsing/parsing_f.py +++ b/api/parsing/parsing_f.py @@ -6,7 +6,7 @@ from api.parsing.hand_parser import parse_query as _parse_query from api.parsing.rewrite import rewrite_query -from api.parsing.spans import QUOTE_CHARS, brace_close_index, find_close_index, opens_regex +from api.parsing.spans import QUOTE_CHARS, brace_close_index, find_close_index, fold_typographic_quotes, opens_regex if TYPE_CHECKING: from api.parsing.nodes import Query @@ -28,6 +28,11 @@ def balance_partial_query(query: str) -> str: The opaque spans never go on it: each one is resolved to its closer and stepped over whole, which is what keeps the quotes, parens and metacharacters inside them from being read as structure. """ + # The balancer and the lexer must agree about which characters are quotes, or a typed opening + # curly quote balances to nothing here and then fails to lex as an unclosed `name:'` after + # parse_query folds it. Same fold, same position: before anything reads a character as a + # delimiter. + query = fold_typographic_quotes(query) open_parens = 0 # Closer for whichever span is still open at the end of the query. Only one is ever needed, # because everything after an unterminated opener is span content — there is nothing left to open, diff --git a/api/parsing/pyparsing_based.py b/api/parsing/pyparsing_based.py index 2266ace76..36eecea79 100644 --- a/api/parsing/pyparsing_based.py +++ b/api/parsing/pyparsing_based.py @@ -45,6 +45,7 @@ flatten_nested_operations, ) from api.parsing.rewrite import rewrite_query +from api.parsing.spans import fold_typographic_quotes if TYPE_CHECKING: from collections.abc import Iterable @@ -588,10 +589,15 @@ def parse_search_query(query: str | None) -> Query: Raises: ValueError: If parsing fails due to syntax errors or invalid operators. """ - original_query = query if query is None or not query.strip(): return Query(TrueNode()) + # The same pre-lex fold `parse_query` applies, for the same reason and in the same position: + # the two parsers must agree about which characters are quotes (test_parser_parity), including + # in the "Failed to parse" message below -- so original_query is captured after the fold, not + # before, and echoes the query the parser actually read rather than the one the user pasted. + query = fold_typographic_quotes(query) + original_query = query query = preprocess_implicit_and(query) expr = get_parse_expr() diff --git a/api/parsing/spans.py b/api/parsing/spans.py index 33d8ddfe4..9e2c2a0e9 100644 --- a/api/parsing/spans.py +++ b/api/parsing/spans.py @@ -11,6 +11,38 @@ QUOTE_CHARS = frozenset("'\"") +# The four typographic quotes Scryfall folds before lexing, and the only four. +# +# Every word processor and phone keyboard turns a typed apostrophe into U+2019 and typed double +# quotes into U+201C/U+201D, so a pasted query carries them constantly -- and this parser read them +# as ordinary letters, which made a query for Gaeas Blessing a search for a card whose name +# contains a curly apostrophe: no rows, no error, no clue. +# +# Measured against api.scryfall.com (2026-08-16) by putting each candidate around a phrase and +# asking whether the phrase searched as ONE term (`o:Xdraw a cardX` -> 2,544 rows means X delimits +# a string). U+2018/U+2019 fold to the ASCII apostrophe and U+201C/U+201D to the ASCII double +# quote; every other quotation-shaped character stays literal and matches nothing -- the guillemets +# (U+00AB/BB, U+2039/203A), the low-9 pair (U+201E, U+201A), the primes (U+2032, U+2033, U+2035), +# the fullwidth quotes (U+FF02, U+FF07), the CJK brackets (U+300C..U+300F), the ornate pairs +# (U+275B..U+275E), backtick, acute, U+02BC. +# +# It is a CHARACTER substitution over the whole query, not a rule about quoted regions: a curly +# apostrophe INSIDE double quotes folds too, which is the only reason `name:"Gaeas +# Blessing"` finds the card; and `name:Gaea"s Blessing` finds nothing, exactly as +# `name:'Gaea"s Blessing'` does. Both directions had to be measured, because folding all four to +# `"` fits the first observation and fails the second. +# +# Lives here rather than in hand_parser, alongside QUOTE_CHARS: the balancer (parsing_f) and both +# parsers all need it applied before they read a single character as a delimiter, and this is +# already the leaf module below all three (see the module docstring). +_TYPOGRAPHIC_QUOTES = str.maketrans({chr(0x2018): "'", chr(0x2019): "'", chr(0x201C): '"', chr(0x201D): '"'}) + + +def fold_typographic_quotes(query: str) -> str: + """Fold the four typographic quotes Scryfall folds; every other character is left alone.""" + return query.translate(_TYPOGRAPHIC_QUOTES) + + # A backslash escapes the character after it inside a quoted string, so '\'' is one string holding a # single quote. Anything that has to find the end of a string has to know that. _ESCAPED_CHAR = re.compile(r"\\(.)", re.DOTALL) diff --git a/api/parsing/tests/test_typographic_quotes.py b/api/parsing/tests/test_typographic_quotes.py new file mode 100644 index 000000000..19f172ffc --- /dev/null +++ b/api/parsing/tests/test_typographic_quotes.py @@ -0,0 +1,97 @@ +"""The four typographic quotes Scryfall folds before lexing, and the only four. + +A word processor or a phone keyboard turns a typed apostrophe into U+2019 and typed double quotes +into U+201C/U+201D, so pasted queries carry them constantly. This parser read them as ordinary +letters, which made a query for Gaea's Blessing (curly apostrophe) a search for a name containing +that curly apostrophe: no rows, no error, no clue. + +Measured against api.scryfall.com (2026-08-16) by putting each candidate around a phrase and +asking whether the phrase searched as ONE term. U+2018/U+2019 fold to `'` and U+201C/U+201D to +`"`; every other quotation-shaped character stays literal -- guillemets, low-9 quotes, primes, +fullwidth forms, CJK brackets, ornate quotes, backtick, acute, U+02BC. +""" + +import pytest + +from api.parsing import generate_sql_query, parse_scryfall_query +from api.parsing.parsing_f import balance_partial_query +from api.parsing.pyparsing_based import parse_search_query +from api.parsing.spans import fold_typographic_quotes + +_LEFT_SINGLE = chr(0x2018) +_RIGHT_SINGLE = chr(0x2019) +_LEFT_DOUBLE = chr(0x201C) +_RIGHT_DOUBLE = chr(0x201D) + +# (query with typographic quotes, the ASCII query it must mean) +FOLDED_CASES = [ + (f"name:{_LEFT_DOUBLE}Gaea{_RIGHT_SINGLE}s Blessing{_RIGHT_DOUBLE}", 'name:"Gaea\'s Blessing"'), + (f"name:{_LEFT_SINGLE}Lightning Bolt{_RIGHT_SINGLE}", "name:'Lightning Bolt'"), + (f"o:{_LEFT_DOUBLE}draw a card{_RIGHT_DOUBLE}", 'o:"draw a card"'), + # The fold is a character substitution over the WHOLE query, not a rule about quoted regions: + # a curly apostrophe INSIDE double quotes folds too, which is what makes Gaea's Blessing + # findable at all. + (f'name:"Gaea{_RIGHT_SINGLE}s Blessing"', 'name:"Gaea\'s Blessing"'), + (f"t:creature o:{_LEFT_DOUBLE}flying{_RIGHT_DOUBLE} c:azorius", 't:creature o:"flying" c:azorius'), +] + + +@pytest.mark.parametrize( + argnames=("query", "canonical_query"), + argvalues=FOLDED_CASES, + ids=[str(i) for i in range(len(FOLDED_CASES))], +) +def test_typographic_quotes_fold(query: str, canonical_query: str) -> None: + """A curly-quoted query parses to exactly what its ASCII-quoted twin parses to, in both parsers.""" + assert generate_sql_query(parse_scryfall_query(query)) == generate_sql_query(parse_scryfall_query(canonical_query)) + assert generate_sql_query(parse_search_query(query)) == generate_sql_query(parse_search_query(canonical_query)) + + +# Quotation-shaped characters Scryfall does NOT fold. Asserted on the fold itself rather than on a +# parse, because several of them are not lexable at all here -- the claim being pinned is that the +# substitution table has exactly four entries, and a wider table is the way this goes wrong. +@pytest.mark.parametrize( + argnames="candidate", + argvalues=[ + chr(0x00AB), # left guillemet + chr(0x00BB), # right guillemet + chr(0x2039), # single left guillemet + chr(0x203A), # single right guillemet + chr(0x201E), # double low-9 + chr(0x201A), # single low-9 + chr(0x2032), # prime + chr(0x2033), # double prime + chr(0x2035), # reversed prime + chr(0xFF02), # fullwidth quotation mark + chr(0xFF07), # fullwidth apostrophe + chr(0x300C), # CJK corner bracket, opening + chr(0x300D), # CJK corner bracket, closing + chr(0x300E), # CJK white corner bracket, opening + chr(0x300F), # CJK white corner bracket, closing + chr(0x275B), # heavy single turned comma quotation mark ornament + chr(0x275C), # heavy single comma quotation mark ornament + chr(0x275D), # heavy double turned comma quotation mark ornament + chr(0x275E), # heavy double comma quotation mark ornament + "`", # grave accent + chr(0x00B4), # acute accent + chr(0x02BC), # modifier letter apostrophe + ], +) +def test_other_quotation_marks_do_not_fold(candidate: str) -> None: + """Everything except the four measured characters is left literal.""" + assert fold_typographic_quotes(f"name:{candidate}Bolt{candidate}") == f"name:{candidate}Bolt{candidate}" + + +@pytest.mark.parametrize( + argnames=("candidate", "folded"), + argvalues=[(_LEFT_SINGLE, "'"), (_RIGHT_SINGLE, "'"), (_LEFT_DOUBLE, '"'), (_RIGHT_DOUBLE, '"')], +) +def test_the_four_that_fold(candidate: str, folded: str) -> None: + """The whole table, one row at a time.""" + assert fold_typographic_quotes(f"a{candidate}b") == f"a{folded}b" + + +def test_balance_folds_before_counting_quotes() -> None: + """The balancer sees the folded text, or a typed opening curly quote balances to nothing.""" + assert balance_partial_query(f"name:{_LEFT_SINGLE}Lightning") == "name:'Lightning'" + assert balance_partial_query(f"o:{_LEFT_DOUBLE}draw") == 'o:"draw"' From c4ecbb8c79bd870d397811819c17686144fe2367 Mon Sep 17 00:00:00 2001 From: Joe Bylund Date: Sat, 22 Aug 2026 14:07:11 -0400 Subject: [PATCH 2/3] Trim the typographic-quotes comment and drop an unverified claim The spans.py comment and the test module docstring both asserted "every word processor and phone keyboard" curly-quotes typed apostrophes -- an overclaim nobody had actually checked, and a 24-line comment for a 4-line function regardless. Cut to the mechanism (which four characters fold, and that the table is measured against api.scryfall.com) and dropped the universal claim about input devices. --- api/parsing/spans.py | 28 +++----------------- api/parsing/tests/test_typographic_quotes.py | 14 ++++------ 2 files changed, 9 insertions(+), 33 deletions(-) diff --git a/api/parsing/spans.py b/api/parsing/spans.py index 9e2c2a0e9..74238a445 100644 --- a/api/parsing/spans.py +++ b/api/parsing/spans.py @@ -11,30 +11,10 @@ QUOTE_CHARS = frozenset("'\"") -# The four typographic quotes Scryfall folds before lexing, and the only four. -# -# Every word processor and phone keyboard turns a typed apostrophe into U+2019 and typed double -# quotes into U+201C/U+201D, so a pasted query carries them constantly -- and this parser read them -# as ordinary letters, which made a query for Gaeas Blessing a search for a card whose name -# contains a curly apostrophe: no rows, no error, no clue. -# -# Measured against api.scryfall.com (2026-08-16) by putting each candidate around a phrase and -# asking whether the phrase searched as ONE term (`o:Xdraw a cardX` -> 2,544 rows means X delimits -# a string). U+2018/U+2019 fold to the ASCII apostrophe and U+201C/U+201D to the ASCII double -# quote; every other quotation-shaped character stays literal and matches nothing -- the guillemets -# (U+00AB/BB, U+2039/203A), the low-9 pair (U+201E, U+201A), the primes (U+2032, U+2033, U+2035), -# the fullwidth quotes (U+FF02, U+FF07), the CJK brackets (U+300C..U+300F), the ornate pairs -# (U+275B..U+275E), backtick, acute, U+02BC. -# -# It is a CHARACTER substitution over the whole query, not a rule about quoted regions: a curly -# apostrophe INSIDE double quotes folds too, which is the only reason `name:"Gaeas -# Blessing"` finds the card; and `name:Gaea"s Blessing` finds nothing, exactly as -# `name:'Gaea"s Blessing'` does. Both directions had to be measured, because folding all four to -# `"` fits the first observation and fails the second. -# -# Lives here rather than in hand_parser, alongside QUOTE_CHARS: the balancer (parsing_f) and both -# parsers all need it applied before they read a single character as a delimiter, and this is -# already the leaf module below all three (see the module docstring). +# Curly quotes (U+2018/U+2019 single, U+201C/U+201D double) are read as ordinary letters otherwise, +# so a pasted `name:'Gaea's Blessing'` (curly apostrophe) silently matches nothing. Only these four +# fold -- everything else quotation-shaped stays literal, matched against what api.scryfall.com +# itself treats as a quote (measured 2026-08-16). _TYPOGRAPHIC_QUOTES = str.maketrans({chr(0x2018): "'", chr(0x2019): "'", chr(0x201C): '"', chr(0x201D): '"'}) diff --git a/api/parsing/tests/test_typographic_quotes.py b/api/parsing/tests/test_typographic_quotes.py index 19f172ffc..98d925ff4 100644 --- a/api/parsing/tests/test_typographic_quotes.py +++ b/api/parsing/tests/test_typographic_quotes.py @@ -1,14 +1,10 @@ """The four typographic quotes Scryfall folds before lexing, and the only four. -A word processor or a phone keyboard turns a typed apostrophe into U+2019 and typed double quotes -into U+201C/U+201D, so pasted queries carry them constantly. This parser read them as ordinary -letters, which made a query for Gaea's Blessing (curly apostrophe) a search for a name containing -that curly apostrophe: no rows, no error, no clue. - -Measured against api.scryfall.com (2026-08-16) by putting each candidate around a phrase and -asking whether the phrase searched as ONE term. U+2018/U+2019 fold to `'` and U+201C/U+201D to -`"`; every other quotation-shaped character stays literal -- guillemets, low-9 quotes, primes, -fullwidth forms, CJK brackets, ornate quotes, backtick, acute, U+02BC. +Curly quotes (U+2018/U+2019 single, U+201C/U+201D double) show up in pasted text and were read as +ordinary letters, silently turning a search for Gaea's Blessing (curly apostrophe) into zero +results. Fold table matched against what api.scryfall.com itself treats as a quote (2026-08-16); +everything else quotation-shaped -- guillemets, low-9 quotes, primes, fullwidth forms, CJK +brackets, ornate quotes, backtick, acute, U+02BC -- stays literal. """ import pytest From 54c71b1dfe17b5976e17fb50de17770e4f6ae53b Mon Sep 17 00:00:00 2001 From: Joe Bylund Date: Sat, 22 Aug 2026 14:17:03 -0400 Subject: [PATCH 3/3] Fix misleading example: the apostrophe in the comment isn't curly Both the spans.py comment and the test docstring wrote out the example as name:'Gaea's Blessing' with a straight ASCII apostrophe, then labeled it "(curly apostrophe)" -- true of the bug, not of the text on the line. Switched to the placeholder already used elsewhere in this file so the example doesn't contradict its own caption. --- api/parsing/spans.py | 6 +++--- api/parsing/tests/test_typographic_quotes.py | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/api/parsing/spans.py b/api/parsing/spans.py index 74238a445..2103cf0c6 100644 --- a/api/parsing/spans.py +++ b/api/parsing/spans.py @@ -12,9 +12,9 @@ QUOTE_CHARS = frozenset("'\"") # Curly quotes (U+2018/U+2019 single, U+201C/U+201D double) are read as ordinary letters otherwise, -# so a pasted `name:'Gaea's Blessing'` (curly apostrophe) silently matches nothing. Only these four -# fold -- everything else quotation-shaped stays literal, matched against what api.scryfall.com -# itself treats as a quote (measured 2026-08-16). +# so a pasted `name:'Gaeas Blessing'` silently matches nothing. Only these four fold -- +# everything else quotation-shaped stays literal, matched against what api.scryfall.com itself +# treats as a quote (measured 2026-08-16). _TYPOGRAPHIC_QUOTES = str.maketrans({chr(0x2018): "'", chr(0x2019): "'", chr(0x201C): '"', chr(0x201D): '"'}) diff --git a/api/parsing/tests/test_typographic_quotes.py b/api/parsing/tests/test_typographic_quotes.py index 98d925ff4..1a42a7929 100644 --- a/api/parsing/tests/test_typographic_quotes.py +++ b/api/parsing/tests/test_typographic_quotes.py @@ -1,10 +1,10 @@ """The four typographic quotes Scryfall folds before lexing, and the only four. Curly quotes (U+2018/U+2019 single, U+201C/U+201D double) show up in pasted text and were read as -ordinary letters, silently turning a search for Gaea's Blessing (curly apostrophe) into zero -results. Fold table matched against what api.scryfall.com itself treats as a quote (2026-08-16); -everything else quotation-shaped -- guillemets, low-9 quotes, primes, fullwidth forms, CJK -brackets, ornate quotes, backtick, acute, U+02BC -- stays literal. +ordinary letters, silently turning a search for Gaeas Blessing into zero results. Fold +table matched against what api.scryfall.com itself treats as a quote (2026-08-16); everything else +quotation-shaped -- guillemets, low-9 quotes, primes, fullwidth forms, CJK brackets, ornate quotes, +backtick, acute, U+02BC -- stays literal. """ import pytest