complete MLT options

Author: ankitml

examples/more_like_this.py

@@ -14,8 +14,6 @@
 - Content discovery and exploration
 """
 
-import json
-
 from _common import MockItem, setup_mock_items
 
 from paradedb.functions import Score
@@ -99,12 +97,12 @@ def demo_similar_by_text() -> None:
     user_description = "comfortable wireless audio for running"
     print(f"\nUser wants: '{user_description}'")
 
-    # MoreLikeThis with text requires JSON format: {"field": "text"}
-    text_json = json.dumps({"description": user_description})
-
+    # MoreLikeThis with text requires fields parameter
     print("\nMatching products:")
     similar = (
-        MockItem.objects.filter(MoreLikeThis(text=text_json))
+        MockItem.objects.filter(
+            MoreLikeThis(text=user_description, fields=["description"])
+        )
         .annotate(score=Score())
         .order_by("-score")[:5]
     )

src/paradedb/search.py

@@ -126,6 +126,9 @@ def __init__(
         min_doc_freq: int | None = None,
         max_term_freq: int | None = None,
         max_doc_freq: int | None = None,
+        min_word_length: int | None = None,
+        max_word_length: int | None = None,
+        stopwords: Iterable[str] | None = None,
     ) -> None:
         super().__init__()
         self.product_id = product_id
@@ -138,6 +141,9 @@ def __init__(
         self.min_doc_freq = min_doc_freq
         self.max_term_freq = max_term_freq
         self.max_doc_freq = max_doc_freq
+        self.min_word_length = min_word_length
+        self.max_word_length = max_word_length
+        self.stopwords = list(stopwords) if stopwords is not None else None
         self._validate()
 
     def _validate(self) -> None:
@@ -246,6 +252,9 @@ def _get_options(self) -> dict[str, object | None]:
             "min_doc_frequency": self.min_doc_freq,
             "max_term_frequency": self.max_term_freq,
             "max_doc_frequency": self.max_doc_freq,
+            "min_word_length": self.min_word_length,
+            "max_word_length": self.max_word_length,
+            "stopwords": self.stopwords,
         }
 
     @staticmethod
@@ -259,7 +268,12 @@ def _render_options(options: dict[str, object | None]) -> str:
         for key, value in options.items():
             if value is None:
                 continue
-            rendered.append(f"{key} => {value}")
+            # Handle stopwords array
+            if key == "stopwords" and isinstance(value, list):
+                quoted = "'" + "','".join(w.replace("'", "''") for w in value) + "'"
+                rendered.append(f"{key} => ARRAY[{quoted}]")
+            else:
+                rendered.append(f"{key} => {value}")
         if not rendered:
             return ""
         return ", " + ", ".join(rendered)

tests/integration/test_paradedb_queries.py

@@ -134,6 +134,127 @@ def test_more_like_this_by_document() -> None:
     assert ids == {12}
 
 
+def test_more_like_this_with_stopwords() -> None:
+    """MLT with stopwords excludes terms from matching - verified by comparing results."""
+    # Get baseline results without stopwords
+    baseline_ids = _ids(
+        MockItem.objects.filter(
+            MoreLikeThis(
+                product_id=3,  # "Sleek running shoes"
+                fields=["description"],
+            )
+        )
+    )
+
+    # Get results with "shoes" as stopword
+    with_stopword_ids = _ids(
+        MockItem.objects.filter(
+            MoreLikeThis(
+                product_id=3,
+                fields=["description"],
+                stopwords=["shoes"],
+            )
+        )
+    )
+
+    # Source item always included in both
+    assert 3 in baseline_ids
+    assert 3 in with_stopword_ids
+
+    # With "shoes" as stopword, results should be different
+    # (fewer matches since "shoes" term is excluded from matching)
+    assert (
+        with_stopword_ids != baseline_ids
+    ), "Stopwords should change the results - excluding 'shoes' should remove shoe-related matches"
+
+    # Typically, stopwords should reduce the number of matches
+    # (unless all matches are from other terms like "running")
+    assert len(with_stopword_ids) <= len(
+        baseline_ids
+    ), "Stopwords should not increase match count"
+
+
+def test_more_like_this_with_word_length() -> None:
+    """MLT with min/max word length filters terms - verified by comparing results."""
+    # Get baseline without word length filter
+    baseline_ids = _ids(
+        MockItem.objects.filter(
+            MoreLikeThis(
+                product_id=3,  # "Sleek running shoes"
+                fields=["description"],
+            )
+        )
+    )
+
+    # Get results with min_word_length=6 (filters out "shoes" which is 5 chars)
+    with_min_length_ids = _ids(
+        MockItem.objects.filter(
+            MoreLikeThis(
+                product_id=3,
+                fields=["description"],
+                min_word_length=6,
+            )
+        )
+    )
+
+    # Source item always included
+    assert 3 in baseline_ids
+    assert 3 in with_min_length_ids
+
+    # Results should differ when short words are filtered
+    assert (
+        with_min_length_ids != baseline_ids
+    ), "min_word_length=6 should filter out 'shoes' (5 chars) and change results"
+
+
+def test_more_like_this_stopwords_reversible() -> None:
+    """Verify stopwords effect is consistent and reversible."""
+    # First query with stopwords
+    ids_with = _ids(
+        MockItem.objects.filter(
+            MoreLikeThis(
+                product_id=3,
+                fields=["description"],
+                stopwords=["shoes"],  # The main matching term
+            )
+        )
+    )
+
+    # Second query without stopwords (baseline)
+    ids_without = _ids(
+        MockItem.objects.filter(
+            MoreLikeThis(
+                product_id=3,
+                fields=["description"],
+            )
+        )
+    )
+
+    # Third query with same stopwords again - should be consistent
+    ids_with_again = _ids(
+        MockItem.objects.filter(
+            MoreLikeThis(
+                product_id=3,
+                fields=["description"],
+                stopwords=["shoes"],
+            )
+        )
+    )
+
+    # Stopwords should produce consistent results (deterministic)
+    assert ids_with == ids_with_again, "Same stopwords should produce same results"
+
+    # With "shoes" as stopword, we should have fewer or equal matches
+    # (excluding the main matching term reduces similarity matches)
+    assert len(ids_with) <= len(
+        ids_without
+    ), "Stopwords should not increase match count"
+
+    # Source item always included regardless of stopwords
+    assert 3 in ids_with
+    assert 3 in ids_without
+
+
 def test_metadata_color_literal_search() -> None:
     """Search over JSON color subfield should return the silver items."""
     with connection.cursor() as cursor:

tests/test_edge_cases.py

@@ -266,9 +266,35 @@ def test_mlt_with_all_options(self) -> None:
             min_doc_freq=1,
             max_term_freq=100,
             max_doc_freq=1000,
+            min_word_length=3,
+            max_word_length=15,
+            stopwords=["the", "a"],
         )
         assert mlt.min_term_freq == 2
         assert mlt.max_query_terms == 10
+        assert mlt.min_word_length == 3
+        assert mlt.max_word_length == 15
+        assert mlt.stopwords == ["the", "a"]
+
+    def test_mlt_stopwords_empty_list(self) -> None:
+        """MLT with empty stopwords list works."""
+        mlt = MoreLikeThis(product_id=1, stopwords=[])
+        assert mlt.stopwords == []
+
+    def test_mlt_stopwords_tuple(self) -> None:
+        """MLT with stopwords as tuple converts to list."""
+        mlt = MoreLikeThis(product_id=1, stopwords=("the", "a", "an"))
+        assert mlt.stopwords == ["the", "a", "an"]
+
+    def test_mlt_word_length_validation(self) -> None:
+        """MLT word length parameters accept integers."""
+        mlt = MoreLikeThis(
+            product_id=1,
+            min_word_length=2,
+            max_word_length=20,
+        )
+        assert isinstance(mlt.min_word_length, int)
+        assert isinstance(mlt.max_word_length, int)
 
 
 class TestScoreEdgeCases:

tests/test_sql_generation.py

@@ -415,6 +415,67 @@ def test_mlt_by_document(self) -> None:
             == 'SELECT "tests_product"."id", "tests_product"."description", "tests_product"."category", "tests_product"."rating", "tests_product"."in_stock", "tests_product"."price", "tests_product"."created_at", "tests_product"."metadata" FROM "tests_product" WHERE "tests_product"."id" @@@ pdb.more_like_this(\'{"description":"running shoes","category":"Footwear"}\')'
         )
 
+    def test_mlt_with_word_length(self) -> None:
+        """MLT with min/max word length parameters."""
+        queryset = Product.objects.filter(
+            MoreLikeThis(product_id=5, min_word_length=3, max_word_length=15)
+        )
+        sql = str(queryset.query)
+        assert "min_word_length => 3" in sql
+        assert "max_word_length => 15" in sql
+        assert "pdb.more_like_this(5," in sql
+
+    def test_mlt_with_stopwords(self) -> None:
+        """MLT with stopwords array parameter."""
+        queryset = Product.objects.filter(
+            MoreLikeThis(product_id=5, stopwords=["the", "a", "an"])
+        )
+        sql = str(queryset.query)
+        assert "stopwords => ARRAY['the','a','an']" in sql
+
+    def test_mlt_with_all_options(self) -> None:
+        """MLT with all available options including new ones."""
+        queryset = Product.objects.filter(
+            MoreLikeThis(
+                product_id=5,
+                fields=["description"],
+                min_term_freq=2,
+                max_query_terms=10,
+                min_doc_freq=1,
+                max_term_freq=100,
+                max_doc_freq=1000,
+                min_word_length=3,
+                max_word_length=20,
+                stopwords=["the", "and", "or"],
+            )
+        )
+        sql = str(queryset.query)
+        assert "min_term_frequency => 2" in sql
+        assert "max_query_terms => 10" in sql
+        assert "min_doc_frequency => 1" in sql
+        assert "max_term_frequency => 100" in sql
+        assert "max_doc_frequency => 1000" in sql
+        assert "min_word_length => 3" in sql
+        assert "max_word_length => 20" in sql
+        assert "stopwords => ARRAY['the','and','or']" in sql
+        assert "ARRAY['description']" in sql
+
+    def test_mlt_text_with_new_options(self) -> None:
+        """MLT with text input and new word length/stopwords options."""
+        queryset = Product.objects.filter(
+            MoreLikeThis(
+                text="comfortable running shoes",
+                fields=["description"],
+                min_word_length=4,
+                max_word_length=12,
+                stopwords=["comfortable"],
+            )
+        )
+        sql = str(queryset.query)
+        assert "min_word_length => 4" in sql
+        assert "max_word_length => 12" in sql
+        assert "stopwords => ARRAY['comfortable']" in sql
+
 
 class TestDjangoIntegration:
     """Test Django ORM integration."""