fix: remove dead IndexExpression config fields and unreachable JSON-key tokenizer error (#136)

Author: philippemnoel

paradedb/indexes.py

@@ -123,15 +123,16 @@ class IndexExpression:
     For text expressions, specify a tokenizer. For non-text expressions
     (integers, timestamps, etc.), omit the tokenizer to use ``pdb.alias``.
 
+    Tokenizer configuration (positional args, named args, token filters,
+    stemmer, etc.) is supplied directly on the ``tokenizer`` object via
+    :class:`~paradedb.search.Tokenizer` factory methods (e.g.
+    ``Tokenizer.simple(options={'stemmer': 'english'})``).
+
     Args:
         expression: A Django expression to index (e.g., ``Lower('title')``).
         alias: Required. The name used to reference this expression in queries.
         tokenizer: Tokenizer for text expressions (e.g., 'simple', 'unicode_words').
             Omit for non-text expressions to use ``pdb.alias``.
-        args: Positional arguments for the tokenizer.
-        named_args: Named arguments for the tokenizer configuration.
-        filters: Token filters (e.g., ['lowercase', 'stemmer']).
-        stemmer: Stemmer language (e.g., 'english').
 
     Example::
 
@@ -162,10 +163,6 @@ class IndexExpression:
     expression: Expression | str
     alias: str
     tokenizer: Tokenizer | None = None
-    args: list[Any] | None = None
-    named_args: dict[str, Any] | None = None
-    filters: list[str] | None = None
-    stemmer: str | None = None
 
 
 class BM25Index(models.Index):
@@ -275,8 +272,7 @@ def _build_index_expressions(
                 if tokenizer is not None or alias is not None:
                     raise ValueError(
                         f"Field {field_name!r} cannot mix 'tokenizers' with "
-                        f"'tokenizer', 'args', 'named_args', 'filters', "
-                        f"'stemmer', or 'alias'."
+                        f"'tokenizer' or 'alias'."
                     )
                 expressions.extend(
                     self._build_multi_tokenizer_expressions(
@@ -373,13 +369,13 @@ def _build_json_key_expressions(
         expressions: list[str] = []
         for key, config in json_keys.items():
             tokenizer = config.get("tokenizer")
-            if not isinstance(tokenizer, Tokenizer):
-                raise TypeError("tokenizer must be a Tokenizer")
             if tokenizer is None:
                 raise ValueError(
                     f"JSON key {key!r} in field {field_name!r} requires an explicit "
                     f"tokenizer (e.g. 'unicode_words', 'simple', 'literal')."
                 )
+            if not isinstance(tokenizer, Tokenizer):
+                raise TypeError("tokenizer must be a Tokenizer")
             key_literal = _quote_term(key)
             expressions.append(f"(({column}->>{key_literal})::{tokenizer.render()})")
         return expressions

tests/test_indexing.py

@@ -52,7 +52,7 @@ def test_tokenizers_mixed_with_top_level_tokenizer_config_raises_value_error() -
         index.create_sql(model=MockItem, schema_editor=DummySchemaEditor())
 
 
-def test_json_key_without_tokenizer_raises_type_error() -> None:
+def test_json_key_without_tokenizer_raises_value_error() -> None:
     index = BM25Index(
         fields={
             "id": {},
@@ -65,6 +65,23 @@ def test_json_key_without_tokenizer_raises_type_error() -> None:
         key_field="id",
         name="mock_items_search_idx",
     )
+    with pytest.raises(ValueError, match="requires an explicit"):
+        index.create_sql(model=MockItem, schema_editor=DummySchemaEditor())
+
+
+def test_json_key_with_invalid_tokenizer_type_raises_type_error() -> None:
+    index = BM25Index(
+        fields={
+            "id": {},
+            "metadata": {
+                "json_keys": {
+                    "color": {"tokenizer": "simple"},
+                }
+            },
+        },
+        key_field="id",
+        name="mock_items_search_idx",
+    )
     with pytest.raises(TypeError, match="tokenizer must be a Tokenizer"):
         index.create_sql(model=MockItem, schema_editor=DummySchemaEditor())
 
@@ -273,7 +290,7 @@ def test_json_field_native_json_fields(self) -> None:
         )
 
     def test_json_key_without_tokenizer_raises(self) -> None:
-        """JSON keys without an explicit tokenizer raise TypeError."""
+        """JSON keys without an explicit tokenizer raise a descriptive ValueError."""
         index = BM25Index(
             fields={
                 "id": {},
@@ -287,7 +304,7 @@ def test_json_key_without_tokenizer_raises(self) -> None:
             name="mock_items_search_idx",
         )
         schema_editor = _schema_editor()
-        with pytest.raises(TypeError, match="tokenizer must be a Tokenizer"):
+        with pytest.raises(ValueError, match="requires an explicit"):
             index.create_sql(model=MockItem, schema_editor=schema_editor)
 
     def test_json_field_literal_alias(self) -> None: