feat: Introduce function-based API for tokenizers

.github/workflows/ci.yml

@@ -56,10 +56,10 @@ jobs:
         run: uv run --no-sync mypy paradedb
 
       - name: Check API coverage
-        run: python3 scripts/check_api_coverage.py
+        run: uv run --no-sync python scripts/check_api_coverage.py
 
       - name: Check API stub sync
-        run: python3 scripts/check_api_stub_sync.py
+        run: uv run --no-sync python scripts/check_api_stub_sync.py
 
       - name: Wheel install smoke test
         run: bash scripts/smoke_wheel_install.sh
@@ -114,7 +114,7 @@ jobs:
 
     services:
       paradedb:
-        image: paradedb/paradedb:0.22.0-pg${{ matrix.postgres-version }}
+        image: paradedb/paradedb:0.23.0-pg${{ matrix.postgres-version }}
         env:
           POSTGRES_USER: postgres
           POSTGRES_PASSWORD: postgres
@@ -171,14 +171,7 @@ jobs:
           CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
 
       - name: Run examples
-        run: |
-          uv run --with "${DJANGO_SPEC}" python examples/quickstart/quickstart.py
-          uv run --with "${DJANGO_SPEC}" python examples/autocomplete/setup.py
-          uv run --with "${DJANGO_SPEC}" python examples/autocomplete/autocomplete.py
-          uv run --with "${DJANGO_SPEC}" python examples/more_like_this/more_like_this.py
-          uv run --with "${DJANGO_SPEC}" python examples/faceted_search/faceted_search.py
-          uv run --with "${DJANGO_SPEC}" python examples/hybrid_rrf/setup.py
-          uv run --with "${DJANGO_SPEC}" python examples/hybrid_rrf/hybrid_rrf.py
+        run: scripts/run_examples.sh
 
       - name: Run RAG example (if API key configured)
         env:

.github/workflows/schema-compat.yml

@@ -38,10 +38,20 @@ jobs:
         with:
           python-version: "3.13"
 
+      - name: Set up uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          enable-cache: true
+
+      - name: Install dependencies
+        run: |
+          uv venv --python "3.13"
+          uv sync --extra dev
+
       - name: Run Schema Compatibility Check
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: python3 scripts/check_schema_compat.py ${{ env.PARADEDB_VERSION }}
+        run: uv run --no-sync python scripts/check_schema_compat.py ${{ env.PARADEDB_VERSION }}
 
   integration-tests:
     name: Integration Tests

CHANGELOG.md

@@ -6,10 +6,17 @@ All notable changes to this project will be documented in this file. The format
 
 ## Unreleased
 
+## [0.6.0] - 2026-03-23
+
+### Added
+
+- Added support for `icu` and `edge_ngram` tokenizers.
+
 ### Changed
 
 - **BREAKING**: Removed the `Empty`, `ParseWithField`, and `Range` query expressions
   from the public search API.
+- **BREAKING**: Updated tokenizers to be configured with a function-based API instead of using strings. Before: `"simple('lowercase=false')` After: `Tokenizer.simple(options={"lowercase": False})`
 
 ## [0.5.0] - 2026-03-23
 
@@ -181,6 +188,7 @@ All notable changes to this project will be documented in this file. The format
 - JSON field key indexing support
 - Full Django ORM integration with `Q` objects and standard filters
 
+[0.6.0]: https://github.com/paradedb/django-paradedb/compare/v0.5.0...v0.6.0
 [0.5.0]: https://github.com/paradedb/django-paradedb/compare/v0.4.0...v0.5.0
 [0.4.0]: https://github.com/paradedb/django-paradedb/compare/v0.3.0...v0.4.0
 [0.3.0]: https://github.com/paradedb/django-paradedb/compare/v0.2.0...v0.3.0

CONTRIBUTING.md

@@ -55,8 +55,8 @@ uv run ruff format .
 uv run mypy paradedb
 
 # Run API/package consistency checks
-python3 scripts/check_api_coverage.py
-python3 scripts/check_api_stub_sync.py
+uv run python scripts/check_api_coverage.py
+uv run python scripts/check_api_stub_sync.py
 bash scripts/smoke_wheel_install.sh
 ```
 

api.json → api.json5

@@ -40,8 +40,8 @@
     "PDB_TYPE_FUZZY": "pdb.fuzzy",
     "PDB_TYPE_SLOP":  "pdb.slop",
     "PDB_TYPE_QUERY": "pdb.query",
+    "PDB_TYPE_ALIAS":              "pdb.alias",
 
-    "PDB_TYPE_TOKENIZER_ALIAS":              "pdb.alias",
     "PDB_TYPE_TOKENIZER_CHINESE_COMPATIBLE": "pdb.chinese_compatible",
     "PDB_TYPE_TOKENIZER_JIEBA":              "pdb.jieba",
     "PDB_TYPE_TOKENIZER_LINDERA":            "pdb.lindera",
@@ -52,6 +52,8 @@
     "PDB_TYPE_TOKENIZER_SIMPLE":             "pdb.simple",
     "PDB_TYPE_TOKENIZER_SOURCE_CODE":        "pdb.source_code",
     "PDB_TYPE_TOKENIZER_UNICODE_WORDS":      "pdb.unicode_words",
-    "PDB_TYPE_TOKENIZER_WHITESPACE":         "pdb.whitespace"
+    "PDB_TYPE_TOKENIZER_WHITESPACE":         "pdb.whitespace",
+    "PDB_TYPE_TOKENIZER_ICU":                "pdb.icu",
+    "PDB_TYPE_TOKENIZER_EDGE_NGRAM":         "pdb.edge_ngram"
   }
 }

apiignore.json → apiignore.json5

@@ -1,5 +1,5 @@
 {
-  "_comment": "Symbols present in the pg_search schema that django-paradedb intentionally does not wrap. Remove an entry here when you add it to api.json. Sections can be either a flat list or grouped object of lists.",
+  "_comment": "Symbols present in the pg_search schema that django-paradedb intentionally does not wrap. Remove an entry here when you add it to api.json5. Sections can be either a flat list or grouped object of lists.",
 
   "operators": [
     "==="
@@ -21,6 +21,9 @@
     "pdb.alias_out_safe",
     "pdb.alias_recv",
     "pdb.alias_send",
+
+    // These functions are tokenizer implementation details that the ORM
+    // doesn't need to interact with
     "pdb.chinese_compatible_in",
     "pdb.chinese_compatible_out",
     "pdb.chinese_compatible_recv",
@@ -73,6 +76,16 @@
     "pdb.whitespace_out",
     "pdb.whitespace_recv",
     "pdb.whitespace_send",
+    "pdb.edge_ngram_in",
+    "pdb.edge_ngram_out",
+    "pdb.edge_ngram_recv",
+    "pdb.edge_ngram_send",
+    "pdb.json_to_edge_ngram",
+    "pdb.jsonb_to_edge_ngram",
+    "pdb.text_array_to_edge_ngram",
+    "pdb.tokenize_edge_ngram",
+    "pdb.uuid_to_edge_ngram",
+    "pdb.varchar_array_to_edge_ngram",
 
     "pdb.bigint_array_to_alias",
     "pdb.bigint_to_alias",
@@ -204,7 +217,6 @@
   ],
 
   "types": [
-    "pdb.icu",
     "pdb.proximityclause"
   ]
 }

examples/autocomplete/models.py

@@ -5,6 +5,7 @@
 
 from paradedb.indexes import BM25Index
 from paradedb.queryset import ParadeDBManager
+from paradedb.search import Tokenizer
 
 configure_django()
 
@@ -29,15 +30,15 @@ class Meta:
                     "id": {},
                     "description": {
                         "tokenizers": [
-                            {"tokenizer": "unicode_words"},
+                            {"tokenizer": Tokenizer.unicode_words()},
                             {
-                                "tokenizer": "ngram",
-                                "args": [3, 8],
-                                "alias": "description_ngram",
+                                "tokenizer": Tokenizer.ngram(
+                                    3, 8, {"alias": "description_ngram"}
+                                ),
                             },
                         ]
                     },
-                    "category": {"tokenizer": "literal", "alias": "category"},
+                    "category": {"tokenizer": Tokenizer.literal({"alias": "category"})},
                 },
                 key_field="id",
                 name="autocomplete_items_idx",

examples/common.py

@@ -12,6 +12,7 @@
 
 from paradedb.indexes import BM25Index
 from paradedb.queryset import ParadeDBManager
+from paradedb.search import Tokenizer
 
 
 def configure_django() -> None:
@@ -94,7 +95,7 @@ def _mock_items_indexes() -> list[BM25Index]:
                 "id": {},
                 "description": {},
                 "rating": {},
-                "category": {"tokenizer": "literal", "alias": "category"},
+                "category": {"tokenizer": Tokenizer.literal({"alias": "category"})},
                 "metadata": {"json_fields": {"fast": True}},
             },
             key_field="id",

paradedb/__init__.py

@@ -40,6 +40,7 @@
     "Snippets",
     "Term",
     "TermSet",
+    "Tokenizer",
     "paradedb_index_segments",
     "paradedb_indexes",
     "paradedb_verify_all_indexes",
@@ -82,6 +83,7 @@
     "RegexPhrase": ("paradedb.search", "RegexPhrase"),
     "Term": ("paradedb.search", "Term"),
     "TermSet": ("paradedb.search", "TermSet"),
+    "Tokenizer": ("paradedb.search", "Tokenizer"),
 }
 
 

paradedb/api.py

@@ -1,14 +1,15 @@
-"""Load ParadeDB SQL API constants from api.json.
+"""Load ParadeDB SQL API constants from api.json5.
 
-To add or modify a symbol, edit api.json only. Every key in every section
+To add or modify a symbol, edit api.json5 only. Every key in every section
 (operators/functions/types) is exposed as a module-level name::
 
     from paradedb.api import FN_ALL, OP_SEARCH, PDB_TYPE_BOOST
 """
 
-import json
 from pathlib import Path
 
+import json5
+
 
 def _validate_api_payload(payload: object) -> dict[str, dict[str, str]]:
     if not isinstance(payload, dict):
@@ -38,19 +39,21 @@ def _validate_api_payload(payload: object) -> dict[str, dict[str, str]]:
 
 
 def _load_api() -> dict[str, dict[str, str]]:
-    # In installed wheels, api.json is bundled into the package directory.
-    packaged_api = Path(__file__).with_name("api.json")
+    # In installed wheels, api.json5 is bundled into the package directory.
+    packaged_api = Path(__file__).with_name("api.json5")
     if packaged_api.is_file():
         return _validate_api_payload(
-            json.loads(packaged_api.read_text(encoding="utf-8"))
+            json5.loads(packaged_api.read_text(encoding="utf-8"))
         )
 
     # In editable/source checkouts, fall back to the repository-root file.
-    source_api = Path(__file__).resolve().parents[1] / "api.json"
+    source_api = Path(__file__).resolve().parents[1] / "api.json5"
     if source_api.is_file():
-        return _validate_api_payload(json.loads(source_api.read_text(encoding="utf-8")))
+        return _validate_api_payload(
+            json5.loads(source_api.read_text(encoding="utf-8"))
+        )
 
-    raise FileNotFoundError("Could not locate api.json for paradedb.api.")
+    raise FileNotFoundError("Could not locate api.json5 for paradedb.api.")
 
 
 _api = _load_api()

paradedb/api.pyi

@@ -36,7 +36,7 @@ PDB_TYPE_CONST: str
 PDB_TYPE_FUZZY: str
 PDB_TYPE_SLOP: str
 PDB_TYPE_QUERY: str
-PDB_TYPE_TOKENIZER_ALIAS: str
+PDB_TYPE_ALIAS: str
 PDB_TYPE_TOKENIZER_CHINESE_COMPATIBLE: str
 PDB_TYPE_TOKENIZER_JIEBA: str
 PDB_TYPE_TOKENIZER_LINDERA: str
@@ -48,3 +48,5 @@ PDB_TYPE_TOKENIZER_SIMPLE: str
 PDB_TYPE_TOKENIZER_SOURCE_CODE: str
 PDB_TYPE_TOKENIZER_UNICODE_WORDS: str
 PDB_TYPE_TOKENIZER_WHITESPACE: str
+PDB_TYPE_TOKENIZER_ICU: str
+PDB_TYPE_TOKENIZER_EDGE_NGRAM: str