feat: Unify tokenizer API and add support for pdb.icu and pdb.edge_ngram (#53)

Author: isaacvando

.github/workflows/ci.yml

@@ -89,7 +89,7 @@ jobs:
 
     services:
       paradedb:
-        image: paradedb/paradedb:0.22.0-pg${{ matrix.postgres-version }}
+        image: paradedb/paradedb:0.23.0-pg${{ matrix.postgres-version }}
         ports:
           - 5432:5432
         env:

.github/workflows/schema-compat.yml

@@ -38,10 +38,15 @@ jobs:
         with:
           python-version: "3.13"
 
+      - name: Set up uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          enable-cache: true
+
       - name: Run Schema Compatibility Check
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: python scripts/check_schema_compat.py ${{ env.PARADEDB_VERSION }}
+        run: uv run --no-sync scripts/check_schema_compat.py ${{ env.PARADEDB_VERSION }}
 
   integration-tests:
     name: Integration Tests

api.json api.json5api.json renamed to api.json5

@@ -52,6 +52,7 @@
     "PDB_TYPE_TOKENIZER_SOURCE_CODE":        "pdb.source_code",
     "PDB_TYPE_TOKENIZER_UNICODE_WORDS":      "pdb.unicode_words",
     "PDB_TYPE_TOKENIZER_WHITESPACE":         "pdb.whitespace",
-    "PDB_TYPE_TOKENIZER_ICU":                "pdb.icu"
+    "PDB_TYPE_TOKENIZER_ICU":                "pdb.icu",
+    "PDB_TYPE_TOKENIZER_EDGE_NGRAM":         "pdb.edge_ngram"
   }
 }

apiignore.json apiignore.json5apiignore.json renamed to apiignore.json5

@@ -19,6 +19,9 @@
     "pdb.alias_out_safe",
     "pdb.alias_recv",
     "pdb.alias_send",
+
+    // These functions are implementation details of tokenizers that the ORM
+    // doesn't need to interact with
     "pdb.chinese_compatible_in",
     "pdb.chinese_compatible_out",
     "pdb.chinese_compatible_recv",
@@ -71,6 +74,16 @@
     "pdb.whitespace_out",
     "pdb.whitespace_recv",
     "pdb.whitespace_send",
+    "pdb.edge_ngram_in",
+    "pdb.edge_ngram_out",
+    "pdb.edge_ngram_recv",
+    "pdb.edge_ngram_send",
+    "pdb.json_to_edge_ngram",
+    "pdb.jsonb_to_edge_ngram",
+    "pdb.text_array_to_edge_ngram",
+    "pdb.tokenize_edge_ngram",
+    "pdb.uuid_to_edge_ngram",
+    "pdb.varchar_array_to_edge_ngram",
 
     "pdb.bigint_array_to_alias",
     "pdb.bigint_to_alias",

examples/autocomplete/setup.py

@@ -6,6 +6,7 @@
 from sqlalchemy.engine import Engine
 from sqlalchemy.orm import DeclarativeBase, Mapped, Session, mapped_column
 
+from paradedb import tokenizer
 from paradedb.sqlalchemy import indexing
 
 
@@ -34,12 +35,12 @@ class Product(Base):
 Index(
     "products_autocomplete_bm25_idx",
     indexing.BM25Field(Product.id),
-    indexing.BM25Field(Product.description, tokenizer=indexing.tokenize.unicode(lowercase=True)),
+    indexing.BM25Field(Product.description),
     indexing.BM25Field(
         Product.description,
-        tokenizer=indexing.tokenize.ngram(min_gram=3, max_gram=8, prefix_only=True, alias="description_ngram"),
+        tokenizer=tokenizer.ngram(3, 8, options={"prefix_only": True, "alias": "description_ngram"}),
     ),
-    indexing.BM25Field(Product.category, tokenizer=indexing.tokenize.literal()),
+    indexing.BM25Field(Product.category, tokenizer=tokenizer.literal()),
     indexing.BM25Field(Product.rating),
     postgresql_using="bm25",
     postgresql_with={"key_field": "id"},

examples/faceted_search/setup.py

@@ -6,6 +6,7 @@
 from sqlalchemy.engine import Engine
 from sqlalchemy.orm import DeclarativeBase, Mapped, Session, mapped_column
 
+from paradedb import tokenizer
 from paradedb.sqlalchemy import indexing
 
 
@@ -34,8 +35,8 @@ class Product(Base):
 Index(
     "products_facets_bm25_idx",
     indexing.BM25Field(Product.id),
-    indexing.BM25Field(Product.description, tokenizer=indexing.tokenize.unicode(lowercase=True)),
-    indexing.BM25Field(Product.category, tokenizer=indexing.tokenize.literal()),
+    indexing.BM25Field(Product.description),
+    indexing.BM25Field(Product.category, tokenizer=tokenizer.literal()),
     indexing.BM25Field(Product.rating),
     postgresql_using="bm25",
     postgresql_with={"key_field": "id"},

examples/hybrid_rrf/setup.py

@@ -6,6 +6,7 @@
 from sqlalchemy.engine import Engine
 from sqlalchemy.orm import DeclarativeBase, Mapped, Session, mapped_column
 
+from paradedb import tokenizer
 from paradedb.sqlalchemy import indexing
 
 
@@ -34,8 +35,8 @@ class Product(Base):
 Index(
     "products_hybrid_rrf_bm25_idx",
     indexing.BM25Field(Product.id),
-    indexing.BM25Field(Product.description, tokenizer=indexing.tokenize.unicode(lowercase=True)),
-    indexing.BM25Field(Product.category, tokenizer=indexing.tokenize.literal()),
+    indexing.BM25Field(Product.description),
+    indexing.BM25Field(Product.category, tokenizer=tokenizer.literal()),
     indexing.BM25Field(Product.rating),
     postgresql_using="bm25",
     postgresql_with={"key_field": "id"},

examples/more_like_this/setup.py

@@ -6,6 +6,7 @@
 from sqlalchemy.engine import Engine
 from sqlalchemy.orm import DeclarativeBase, Mapped, Session, mapped_column
 
+from paradedb import tokenizer
 from paradedb.sqlalchemy import indexing
 
 
@@ -34,8 +35,8 @@ class Product(Base):
 Index(
     "products_mlt_bm25_idx",
     indexing.BM25Field(Product.id),
-    indexing.BM25Field(Product.description, tokenizer=indexing.tokenize.unicode(lowercase=True)),
-    indexing.BM25Field(Product.category, tokenizer=indexing.tokenize.literal()),
+    indexing.BM25Field(Product.description),
+    indexing.BM25Field(Product.category, tokenizer=tokenizer.literal()),
     indexing.BM25Field(Product.rating),
     postgresql_using="bm25",
     postgresql_with={"key_field": "id"},

examples/quickstart/setup.py

@@ -6,6 +6,7 @@
 from sqlalchemy.engine import Engine
 from sqlalchemy.orm import DeclarativeBase, Mapped, Session, mapped_column
 
+from paradedb import tokenizer
 from paradedb.sqlalchemy import indexing
 
 
@@ -34,8 +35,8 @@ class Product(Base):
 Index(
     "products_bm25_idx",
     indexing.BM25Field(Product.id),
-    indexing.BM25Field(Product.description, tokenizer=indexing.tokenize.unicode(lowercase=True)),
-    indexing.BM25Field(Product.category, tokenizer=indexing.tokenize.literal()),
+    indexing.BM25Field(Product.description),
+    indexing.BM25Field(Product.category, tokenizer=tokenizer.literal()),
     indexing.BM25Field(Product.rating),
     postgresql_using="bm25",
     postgresql_with={"key_field": "id"},

examples/rag/setup.py

@@ -30,7 +30,7 @@ class Document(Base):
 Index(
     "documents_bm25_idx",
     indexing.BM25Field(Document.id),
-    indexing.BM25Field(Document.content, tokenizer=indexing.tokenize.unicode(lowercase=True)),
+    indexing.BM25Field(Document.content),
     postgresql_using="bm25",
     postgresql_with={"key_field": "id"},
 )