Remove index_schema API and prune legacy tokenizer config branch

Author: ankitml

README.md

@@ -70,6 +70,39 @@ products_bm25_idx = Index(
 )
 ```
 
+Tokenizer configs can use a Django/Rails-style structured shape:
+
+```python
+products_bm25_idx = Index(
+    "products_bm25_idx",
+    indexing.BM25Field(Product.id),
+    indexing.BM25Field(
+        Product.description,
+        tokenizer=indexing.tokenize.from_config(
+            {
+                "tokenizer": "simple",
+                "filters": ["lowercase", "stemmer"],
+                "stemmer": "english",
+                "alias": "description_simple",
+            }
+        ),
+    ),
+    indexing.BM25Field(
+        Product.description,
+        tokenizer=indexing.tokenize.from_config(
+            {
+                "tokenizer": "ngram",
+                "args": [3, 8],
+                "named_args": {"prefix_only": True},
+                "alias": "description_ngram",
+            }
+        ),
+    ),
+    postgresql_using="bm25",
+    postgresql_with={"key_field": "id"},
+)
+```
+
 ## Query APIs
 
 - Basic predicates: `match_all`, `match_any`, `term`, `phrase`, `fuzzy`, `regex`, `all`
@@ -108,8 +141,24 @@ Usage:
 
 ```python
 op.create_bm25_index("products_bm25_idx", "products", ["id", "description"], key_field="id")
+# Tokenizer-aware expressions are supported too:
+op.create_bm25_index(
+    "products_bm25_idx",
+    "products",
+    ["id", "((description)::pdb.simple('alias=description_simple,lowercase=true'))"],
+    key_field="id",
+)
+# Schema-aware operations:
+op.create_bm25_index(
+    "products_bm25_idx",
+    "products",
+    ["id", "description"],
+    key_field="id",
+    table_schema="search",
+)
 op.reindex_bm25("products_bm25_idx", concurrently=True)
-op.drop_bm25_index("products_bm25_idx", if_exists=True)
+op.reindex_bm25("products_bm25_idx", concurrently=True, schema="search")
+op.drop_bm25_index("products_bm25_idx", if_exists=True, schema="search")
 ```
 
 ## Validation and Guardrails

paradedb/sqlalchemy/alembic.py

@@ -35,14 +35,12 @@ def __init__(
         key_field: str,
         *,
         table_schema: str | None = None,
-        index_schema: str | None = None,
     ) -> None:
         self.index_name = index_name
         self.table_name = table_name
         self.expressions = expressions
         self.key_field = key_field
         self.table_schema = table_schema
-        self.index_schema = index_schema
 
     @classmethod
     def create_bm25_index(
@@ -54,7 +52,6 @@ def create_bm25_index(
         *,
         key_field: str,
         table_schema: str | None = None,
-        index_schema: str | None = None,
     ) -> MigrateOperation:
         return operations.invoke(
             cls(
@@ -63,7 +60,6 @@ def create_bm25_index(
                 expressions,
                 key_field,
                 table_schema=table_schema,
-                index_schema=index_schema,
             )
         )
 
@@ -89,8 +85,6 @@ def _render_create_bm25_index_op(autogen_context, op: CreateBM25IndexOp) -> str:
     ]
     if op.table_schema is not None:
         parts.append(f"table_schema={op.table_schema!r}")
-    if op.index_schema is not None:
-        parts.append(f"index_schema={op.index_schema!r}")
     return f"op.create_bm25_index({', '.join(parts)})"
 
 
@@ -298,7 +292,6 @@ def _compare_bm25_indexes(autogen_context, upgrade_ops, schemas) -> PriorityDisp
                     expressions=expressions,
                     key_field=key_field,
                     table_schema=key[0],
-                    index_schema=key[0],
                 )
             )
         else:
@@ -313,7 +306,6 @@ def _compare_bm25_indexes(autogen_context, upgrade_ops, schemas) -> PriorityDisp
                         expressions=expressions,
                         key_field=key_field,
                         table_schema=key[0],
-                        index_schema=key[0],
                     )
                 )
 

paradedb/sqlalchemy/indexing.py

@@ -306,8 +306,6 @@ def custom(
 def from_config(config: Mapping[str, Any]) -> TokenizerSpec:
     if not isinstance(config, Mapping):
         raise InvalidArgumentError("tokenizer config must be a mapping")
-    if "options" in config:
-        raise InvalidArgumentError("tokenizer config uses deprecated 'options'. Use 'named_args' instead.")
 
     allowed_keys = {"tokenizer", "args", "named_args", "filters", "stemmer", "alias"}
     unknown = set(config.keys()) - allowed_keys

tests/integration/test_alembic_integration.py

@@ -99,7 +99,6 @@ def test_alembic_create_reindex_drop_with_schema(engine):
             ["id", "description"],
             key_field="id",
             table_schema=schema,
-            index_schema=schema,
         )
 
         exists = conn.execute(
@@ -378,7 +377,6 @@ def test_autogenerate_emits_schema_on_create_for_non_default_schema(engine):
         create_ops = [op for op in upgrade_ops.ops if isinstance(op, pdb_alembic.CreateBM25IndexOp)]
         op = next(o for o in create_ops if o.index_name == _AG_SCHEMA_IDX)
         assert op.table_schema == _AG_SCHEMA
-        assert op.index_schema == _AG_SCHEMA
     finally:
         with engine.begin() as conn:
             conn.execute(text(f'DROP SCHEMA IF EXISTS "{_AG_SCHEMA}" CASCADE'))

tests/unit/test_alembic_unit.py

@@ -66,7 +66,6 @@ def test_create_drop_reindex_sql_generation_with_schema():
         expressions=["id", "description"],
         key_field="id",
         table_schema="analytics",
-        index_schema="analytics",
     )
     pdb_alembic._create_bm25_index_impl(ops, create_op)
     assert ops.sql[-1] == (
@@ -120,11 +119,10 @@ def test_alembic_renderers_registered_and_emit_python():
             expressions=["id", "description"],
             key_field="id",
             table_schema="analytics",
-            index_schema="analytics",
         ),
     )
     assert create_lines_with_schema == [
-        "op.create_bm25_index('products_bm25_idx', 'products', ['id', 'description'], key_field='id', table_schema='analytics', index_schema='analytics')"
+        "op.create_bm25_index('products_bm25_idx', 'products', ['id', 'description'], key_field='id', table_schema='analytics')"
     ]
 
 

tests/unit/test_indexing_unit.py

@@ -257,8 +257,8 @@ def test_tokenizer_from_config_unknown_key_raises():
         tokenize.from_config({"tokenizer": "simple", "unknown": True})
 
 
-def test_tokenizer_from_config_options_deprecated_raises():
-    with pytest.raises(InvalidArgumentError, match="deprecated 'options'"):
+def test_tokenizer_from_config_options_key_raises_as_unknown():
+    with pytest.raises(InvalidArgumentError, match="Unknown tokenizer config keys"):
         tokenize.from_config({"tokenizer": "simple", "options": {"lowercase": True}})