Use json5 for api.json and apiignore.json

Author: isaacvando

.github/workflows/schema-compat.yml

@@ -38,10 +38,15 @@ jobs:
         with:
           python-version: "3.13"
 
+      - name: Set up uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          enable-cache: true
+
       - name: Run Schema Compatibility Check
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: python scripts/check_schema_compat.py ${{ env.PARADEDB_VERSION }}
+        run: uv run --no-sync scripts/check_schema_compat.py ${{ env.PARADEDB_VERSION }}
 
   integration-tests:
     name: Integration Tests

api.json api.json5api.json renamed to api.json5

@@ -19,6 +19,9 @@
     "pdb.alias_out_safe",
     "pdb.alias_recv",
     "pdb.alias_send",
+
+    // These functions are implementation details of tokenizers that the ORM
+    // doesn't need to interact with
     "pdb.chinese_compatible_in",
     "pdb.chinese_compatible_out",
     "pdb.chinese_compatible_recv",

apiignore.json apiignore.json5apiignore.json renamed to apiignore.json5

@@ -52,11 +52,13 @@ test = [
   "pytest-cov>=6.0",
   "psycopg[binary]>=3.1.18",
   "alembic>=1.13.0",
+  "json5>=0.9",
 ]
 dev = [
   "ruff>=0.15.0",
   "mypy>=1.19.0",
   "pre-commit>=4.0.0",
+  "json5>=0.9",
 ]
 
 [tool.pytest.ini_options]

pyproject.toml

@@ -1,27 +1,27 @@
-#!/usr/bin/env python3
-"""Validate that api.json matches the SQLAlchemy wrapper surface."""
+"""Validate that api.json5 matches the SQLAlchemy wrapper surface."""
 
 from __future__ import annotations
 
 import ast
-import json
 import re
 import sys
 from pathlib import Path
 
+import json5
+
 ROOT = Path(__file__).resolve().parents[1]
-API_JSON = ROOT / "api.json"
-APIIGNORE_JSON = ROOT / "apiignore.json"
+API_JSON = ROOT / "api.json5"
+APIIGNORE_JSON = ROOT / "apiignore.json5"
 PDB_SYMBOL_RE = re.compile(r"\bpdb\.[A-Za-z_][A-Za-z0-9_]*\b")
 
 
 def load_json(path: Path) -> object:
     try:
-        return json.loads(path.read_text(encoding="utf-8"))
+        return json5.loads(path.read_text(encoding="utf-8"))
     except FileNotFoundError as exc:
         raise FileNotFoundError(f"{path} not found") from exc
-    except json.JSONDecodeError as exc:
-        raise ValueError(f"invalid JSON in {path}: {exc}") from exc
+    except ValueError as exc:
+        raise ValueError(f"invalid JSON5 in {path}: {exc}") from exc
 
 
 def flatten_ignore(section: object, *, kind: str) -> set[str]:
@@ -141,23 +141,23 @@ def main() -> int:
         return 1
 
     if not isinstance(api, dict):
-        print("❌ api.json must contain a JSON object.", file=sys.stderr)
+        print("❌ api.json5 must contain a JSON object.", file=sys.stderr)
         return 1
     if not isinstance(apiignore, dict):
-        print("❌ apiignore.json must contain a JSON object.", file=sys.stderr)
+        print("❌ apiignore.json5 must contain a JSON object.", file=sys.stderr)
         return 1
 
     try:
         operators = api["operators"]
         functions = api["functions"]
         types = api["types"]
     except KeyError as exc:
-        print(f"❌ api.json missing required section: {exc}", file=sys.stderr)
+        print(f"❌ api.json5 missing required section: {exc}", file=sys.stderr)
         return 1
 
     if not all(isinstance(section, dict) for section in (operators, functions, types)):
         print(
-            "❌ api.json sections operators/functions/types must all be objects.",
+            "❌ api.json5 sections operators/functions/types must all be objects.",
             file=sys.stderr,
         )
         return 1
@@ -201,14 +201,16 @@ def main() -> int:
 
     issues: list[str] = []
     if missing_pdb_symbols:
-        issues.append("api.json pdb.* symbols not referenced by SQLAlchemy wrappers: " + ", ".join(missing_pdb_symbols))
+        issues.append(
+            "api.json5 pdb.* symbols not referenced by SQLAlchemy wrappers: " + ", ".join(missing_pdb_symbols)
+        )
     if missing_operator_symbols:
         issues.append(
-            "api.json operators not referenced by SQLAlchemy wrappers: " + ", ".join(missing_operator_symbols)
+            "api.json5 operators not referenced by SQLAlchemy wrappers: " + ", ".join(missing_operator_symbols)
         )
     if untracked_symbols:
         issues.append(
-            "pdb.* symbols used in package source but missing from api.json/apiignore.json: "
+            "pdb.* symbols used in package source but missing from api.json5/apiignore.json5: "
             + ", ".join(untracked_symbols)
         )
 
@@ -217,7 +219,7 @@ def main() -> int:
         for issue in issues:
             print(f"   - {issue}", file=sys.stderr)
         print(
-            "\nUpdate api.json, apiignore.json, or the SQLAlchemy wrappers so they stay in sync.",
+            "\nUpdate api.json5, apiignore.json5, or the SQLAlchemy wrappers so they stay in sync.",
             file=sys.stderr,
         )
         return 1

scripts/check_api_coverage.py

@@ -1,32 +1,32 @@
-#!/usr/bin/env python3
 """
-Check compatibility between sqlalchemy-paradedb's api.json and a released pg_search schema.
+Check compatibility between sqlalchemy-paradedb's api.json5 and a released pg_search schema.
 
 The schema is downloaded from the corresponding ParadeDB GitHub release and
 checked in both directions:
 
-  Forward:  every symbol in api.json is present in the schema (detects removals/renames).
-  Reverse:  every pdb.* symbol in the schema is either in api.json or in apiignore.json
+  Forward:  every symbol in api.json5 is present in the schema (detects removals/renames).
+  Reverse:  every pdb.* symbol in the schema is either in api.json5 or in apiignore.json5
             (surfaces new paradedb APIs that haven't been wrapped yet).
 
 Example:
     python scripts/check_schema_compat.py 0.22.0
 
-The ignore list is read automatically from apiignore.json (repo root) if it exists.
+The ignore list is read automatically from apiignore.json5 (repo root) if it exists.
 """
 
 import argparse
-import json
 import re
 import shutil
 import subprocess
 import sys
 import tempfile
 from pathlib import Path
 
+import json5
+
 _ROOT_DIR = Path(__file__).resolve().parent.parent
-_API_FILE = _ROOT_DIR / "api.json"
-_IGNORE_FILE = _ROOT_DIR / "apiignore.json"
+_API_FILE = _ROOT_DIR / "api.json5"
+_IGNORE_FILE = _ROOT_DIR / "apiignore.json5"
 _SCHEMA_FILE_NAME = "pg_search.schema.sql"
 
 
@@ -36,8 +36,8 @@ def normalize(sql: str) -> str:
 
 
 def extract_from_api(path: Path) -> dict:
-    """Read api.json and return {functions: [...], operators: [...], types: [...]}."""
-    data = json.loads(path.read_text())
+    """Read api.json5 and return {functions: [...], operators: [...], types: [...]}."""
+    data = json5.loads(path.read_text())
     return {
         "functions": sorted(set(data["functions"].values())),
         "operators": sorted(set(data["operators"].values())),
@@ -104,7 +104,7 @@ def normalize_version(version: str) -> str:
 
 def parse_args(argv: list[str]) -> argparse.Namespace:
     parser = argparse.ArgumentParser(
-        description=("Download pg_search.schema.sql for a ParadeDB release and check it against this repo's api.json.")
+        description=("Download pg_search.schema.sql for a ParadeDB release and check it against this repo's api.json5.")
     )
     parser.add_argument("version", help="ParadeDB version to check, for example 0.22.0")
     return parser.parse_args(argv)
@@ -164,17 +164,17 @@ def run_checks(schema_path: Path, api_path: Path) -> int:
         print(f"❌ Schema file not found: {schema_path}", file=sys.stderr)
         return 1
     if not api_path.exists():
-        print(f"❌ api.json not found: {api_path}", file=sys.stderr)
+        print(f"❌ api.json5 not found: {api_path}", file=sys.stderr)
         return 1
 
     schema = normalize(schema_path.read_text())
     deps = extract_from_api(api_path)
-    ignored = json.loads(_IGNORE_FILE.read_text()) if _IGNORE_FILE.exists() else {}
+    ignored = json5.loads(_IGNORE_FILE.read_text()) if _IGNORE_FILE.exists() else {}
 
     rc = 0
 
     # ------------------------------------------------------------------
-    # Forward check: every symbol in api.json must exist in the schema.
+    # Forward check: every symbol in api.json5 must exist in the schema.
     # ------------------------------------------------------------------
     missing: list[tuple[str, str]] = []
     for fn in deps.get("functions", []):
@@ -189,20 +189,20 @@ def run_checks(schema_path: Path, api_path: Path) -> int:
 
     total_api = sum(len(v) for v in deps.values() if isinstance(v, list))
     if missing:
-        print(f"❌ Forward check: {len(missing)}/{total_api} api.json symbols missing from schema:")
+        print(f"❌ Forward check: {len(missing)}/{total_api} api.json5 symbols missing from schema:")
         for kind, name in missing:
             print(f"   {kind}: {name}")
         print(
             "\nThese symbols were removed or renamed in this version of pg_search.\n"
-            "Update sqlalchemy-paradedb to handle the API change, then update api.json."
+            "Update sqlalchemy-paradedb to handle the API change, then update api.json5."
         )
         rc = 1
     else:
-        print(f"✅ Forward check: all {total_api} api.json symbols present in schema.")
+        print(f"✅ Forward check: all {total_api} api.json5 symbols present in schema.")
 
     # ------------------------------------------------------------------
-    # Reverse check: every pdb.* symbol in the schema must be in api.json
-    # or explicitly ignored in apiignore.json.
+    # Reverse check: every pdb.* symbol in the schema must be in api.json5
+    # or explicitly ignored in apiignore.json5.
     # ------------------------------------------------------------------
     schema_symbols = scan_schema_symbols(schema)
     uncovered: list[tuple[str, str]] = []
@@ -215,12 +215,12 @@ def run_checks(schema_path: Path, api_path: Path) -> int:
 
     total_schema = sum(len(v) for v in schema_symbols.values())
     if uncovered:
-        print(f"\n⚠️  Reverse check: {len(uncovered)} schema symbols not covered by api.json:")
+        print(f"\n⚠️  Reverse check: {len(uncovered)} schema symbols not covered by api.json5:")
         for kind, name in uncovered:
             print(f"   {kind}: {name}")
         print(
             "\nThese are paradedb APIs not yet wrapped by sqlalchemy-paradedb.\n"
-            "Either add them to api.json or add them to apiignore.json."
+            "Either add them to api.json5 or add them to apiignore.json5."
         )
         rc = 1
     else:

scripts/check_schema_compat.py

@@ -185,5 +185,5 @@ def test_all_tokenizers(session, expected: str, tokenizer: str, tokenizer_params
         _sql(stmt)
         == f"""SELECT products.id
 FROM products
-WHERE products.description &&& 'running shoes'::pdb.{expected} ORDER BY products.id"""
+WHERE products.description &&& 'running shoes'::{expected} ORDER BY products.id"""
     )