fix(bigquery): escape all single quotes in literal_string (r-605) (#2791)

BigQuerySqlDialect.literal_string wraps string values in triple quotes
('''...''') but only escaped exact ''' runs. That left a bare ''' inside
the content whenever a value ends in a single quote (its trailing quote
merges with the closing ''' -> '''') or contains a run of 4/5/7/8 single
quotes. The stray ''' prematurely closes the string, so BigQuery sees two
adjacent literals and rejects the statement with "Syntax error:
concatenated string literals must be separated by whitespace or comments".

This surfaced on the AST-fallback check-results INSERT: the failed_rows_query
value embeds the scan-id as its own literal ('''<scan_id>'''), and the
shipped escaping doubled those quotes into a six-quote run.

Escape the backslash first, then EVERY single quote as \', keeping the
triple-quote wrapper so multi-line check SQL still needs no newline
escaping. Escaping every quote guarantees no bare ''' can appear in the
content, for all inputs (verified with a round-trip decode across boundary
quotes, runs 1-8, multiline and backslashes).

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: Niels-b

soda-bigquery/src/soda_bigquery/common/data_sources/bigquery_data_source.py

@@ -230,10 +230,16 @@ def sql_expr_timestamp_add_day(self, timestamp_literal: str) -> str:
     def literal_string(self, value: str) -> Optional[str]:
         if value is None:
             return None
-        # BigQuery triple-quoted strings ('''...''') allow multiline content and
-        # embedded single quotes without escaping.  The only sequence that needs
-        # escaping inside triple-quoted strings is ''' itself and backslashes.
-        escaped = value.replace("\\", "\\\\").replace("'''", "\\'''")
+        # BigQuery triple-quoted strings ('''...''') allow multiline content, so the
+        # wrapper keeps raw newlines in multi-line check SQL from needing escaping.
+        # Escape the backslash first, then EVERY single quote as \'. Escaping every
+        # quote guarantees that no unescaped ''' can appear inside the content and
+        # prematurely close the string (which would make BigQuery see two adjacent
+        # string literals -> "concatenated string literals must be separated by
+        # whitespace or comments"). Only escaping ''' runs was not enough: values
+        # ending in a quote, or containing runs of 4/5/7/8 quotes, still produced a
+        # bare ''' at or near the wrapper boundary.
+        escaped = value.replace("\\", "\\\\").replace("'", "\\'")
         return "'''" + escaped + "'''"
 
     def build_cte_values_sql(self, values: VALUES, alias_columns: list[COLUMN] | None) -> str:

soda-bigquery/tests/unit/test_bigquery_dialect.py

@@ -1,3 +1,4 @@
+import pytest
 from soda_bigquery.common.data_sources.bigquery_data_source import BigQuerySqlDialect
 from soda_core.common.sql_dialect import FROM, RANDOM, SELECT
 
@@ -50,3 +51,89 @@ def test_extract_numeric_precision_case_insensitive():
     assert dialect.extract_numeric_precision(row, columns=[]) == 38
     row = ("col", "BIGNUMERIC")
     assert dialect.extract_numeric_precision(row, columns=[]) == 76
+
+
+# ---------------------------------------------------------------------------
+# BigQuery string literal escaping — literal_string wraps values in triple
+# single quotes ('''...'''). If a bare ''' survives inside the content it
+# prematurely closes the string and BigQuery rejects the statement with
+# "Syntax error: concatenated string literals must be separated by whitespace
+# or comments". This happened for values ending in a quote (boundary merges
+# with the closing ''') and for internal runs of 4/5/7/8 quotes. Regression
+# for the AST-fallback check-result INSERT reported for BigQuery.
+# ---------------------------------------------------------------------------
+
+
+def _decode_bq_triple_quoted(sql_literal: str) -> tuple[bool, str]:
+    """Tokenise a BigQuery '''...''' string literal the way BigQuery does.
+
+    A backslash escapes the next character; a ``'''`` closes the string.
+    Returns ``(closes_exactly_at_end, decoded_value)`` — ``closes_exactly_at_end``
+    is False when the literal never closes OR closes before the final character
+    (i.e. trailing content would be parsed as a second, adjacent literal).
+    """
+    assert sql_literal.startswith("'''") and len(sql_literal) >= 6
+    escapes = {"n": "\n", "t": "\t", "r": "\r"}
+    i, n, out = 3, len(sql_literal), []
+    while i < n:
+        c = sql_literal[i]
+        if c == "\\":
+            nxt = sql_literal[i + 1] if i + 1 < n else ""
+            out.append(escapes.get(nxt, nxt))
+            i += 2
+            continue
+        if c == "'" and sql_literal[i : i + 3] == "'''":
+            return (i + 3 == n, "".join(out))
+        out.append(c)
+        i += 1
+    return (False, "".join(out))
+
+
+@pytest.mark.parametrize(
+    "value",
+    [
+        "",
+        "no quotes here",
+        "'",
+        "''",
+        "'''",
+        "abc'",  # trailing single quote -> boundary '''' before the fix
+        "abc''",  # trailing double quote -> boundary ''''' before the fix
+        "'abc",  # leading quote
+        "SELECT 'x'",  # ends in a quoted literal (the common customer trigger)
+        "x'y",
+        "x''y",
+        "x'''y",
+        "x''''y",  # internal run of 4 -> broke before the fix
+        "x'''''y",  # 5
+        "x''''''y",  # 6
+        "x'''''''y",  # 7
+        "x''''''''y",  # 8
+        "multi\nline 'quoted'\nvalue",  # multiline must survive the triple-quote wrapper
+        "back\\slash and 'quote'",  # backslash + quote
+        "WITH a AS (SELECT 'Zakenadres') SELECT 'missing'",  # customer-style query
+    ],
+)
+def test_literal_string_is_well_formed_and_roundtrips(value: str):
+    dialect = BigQuerySqlDialect()
+    sql_literal = dialect.literal_string(value)
+    closes_at_end, decoded = _decode_bq_triple_quoted(sql_literal)
+    assert closes_at_end, f"literal not closed exactly at end (premature '''): {sql_literal!r}"
+    assert decoded == value, f"round-trip mismatch: {decoded!r} != {value!r} (literal {sql_literal!r})"
+
+
+def test_literal_string_embedded_scan_id_query_roundtrips():
+    """A failed-rows query that itself embeds a scan-id literal ('''...''') is a
+    real check-result value; wrapping it again must stay well-formed."""
+    dialect = BigQuerySqlDialect()
+    scan_id = "481214e3-a379-49f2-9edf-87fdb0d4e73b"
+    inner = dialect.literal_string(scan_id)  # '''481214...'''
+    failed_rows_query = f"SELECT * FROM `t` AS `FAILED_ROWS` WHERE `__soda_scan_id` = {inner};"
+    sql_literal = dialect.literal_string(failed_rows_query)
+    closes_at_end, decoded = _decode_bq_triple_quoted(sql_literal)
+    assert closes_at_end, f"premature ''' in {sql_literal!r}"
+    assert decoded == failed_rows_query
+
+
+def test_literal_string_none_returns_none():
+    assert BigQuerySqlDialect().literal_string(None) is None