Remove mentions of tantivy

Author: prrao87

docs/snippets/integrations.mdx

@@ -50,7 +50,7 @@ export const PyFrameworksAgnoCliChat = "agent.print_response(\n    \"Summarize t
 
 export const PyFrameworksAgnoIngestYoutube = "youtube_url = \"https://www.youtube.com/watch?v=wl6mFyXoxos\"\nvideo_id = extract_video_id(youtube_url)\nytt = YouTubeTranscriptApi()\ntranscript_segments = ytt.fetch(video_id, languages=[\"en\", \"en-US\"]).to_raw_data()\ntranscript_text = \" \".join(segment[\"text\"] for segment in transcript_segments)\n\nknowledge.insert(\n    name=f\"YouTube Transcript ({video_id})\",\n    text_content=transcript_text,\n    metadata={\"source\": \"youtube\", \"video_id\": video_id, \"video_url\": youtube_url},\n)\n";
 
-export const PyFrameworksAgnoSetup = "import os\nimport re\n\nfrom agno.agent import Agent\nfrom agno.knowledge.embedder.openai import OpenAIEmbedder\nfrom agno.knowledge.knowledge import Knowledge\nfrom agno.models.openai import OpenAIResponses\nfrom agno.vectordb.lancedb import LanceDb, SearchType\nfrom youtube_transcript_api import YouTubeTranscriptApi\n\nif \"OPENAI_API_KEY\" not in os.environ:\n    os.environ[\"OPENAI_API_KEY\"] = \"sk-...\"\n\ndef extract_video_id(youtube_url: str) -> str:\n    match = re.search(r\"(?<=v=)[\\w-]+\", youtube_url) or re.search(\n        r\"(?<=be/)[\\w-]+\", youtube_url\n    )\n    if not match:\n        raise ValueError(\"Could not parse YouTube video ID from URL\")\n    return match.group(0)\n\nknowledge = Knowledge(\n    vector_db=LanceDb(\n        uri=\"./tmp/lancedb\",\n        table_name=\"youtube_transcripts\",\n        search_type=SearchType.hybrid,\n        use_tantivy=False,\n        embedder=OpenAIEmbedder(id=\"text-embedding-3-small\"),\n    ),\n)\n";
+export const PyFrameworksAgnoSetup = "import os\nimport re\n\nfrom agno.agent import Agent\nfrom agno.knowledge.embedder.openai import OpenAIEmbedder\nfrom agno.knowledge.knowledge import Knowledge\nfrom agno.models.openai import OpenAIResponses\nfrom agno.vectordb.lancedb import LanceDb, SearchType\nfrom youtube_transcript_api import YouTubeTranscriptApi\n\nif \"OPENAI_API_KEY\" not in os.environ:\n    os.environ[\"OPENAI_API_KEY\"] = \"sk-...\"\n\ndef extract_video_id(youtube_url: str) -> str:\n    match = re.search(r\"(?<=v=)[\\w-]+\", youtube_url) or re.search(\n        r\"(?<=be/)[\\w-]+\", youtube_url\n    )\n    if not match:\n        raise ValueError(\"Could not parse YouTube video ID from URL\")\n    return match.group(0)\n\nknowledge = Knowledge(\n    vector_db=LanceDb(\n        uri=\"./tmp/lancedb\",\n        table_name=\"youtube_transcripts\",\n        search_type=SearchType.hybrid,\n        embedder=OpenAIEmbedder(id=\"text-embedding-3-small\"),\n    ),\n)\n";
 
 export const PyFrameworksLangchainAddImages = "image_uris = [\"./assets/image-1.png\", \"./assets/image-2.png\"]\nvector_store.add_images(uris=image_uris)\n# here image_uris are local fs paths to the images.\n";
 

docs/snippets/search.mdx

@@ -1,6 +1,6 @@
 {/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */}
 
-export const PyBasicFts = "uri = \"data/sample-lancedb\"\ndb = lancedb.connect(uri)\n\ntable = db.create_table(\n    \"my_table_fts\",\n    data=[\n        {\"vector\": [3.1, 4.1], \"text\": \"Frodo was a happy puppy\"},\n        {\"vector\": [5.9, 26.5], \"text\": \"There are several kittens playing\"},\n    ],\n    mode=\"overwrite\",\n)\n\n# passing `use_tantivy=False` to use lance FTS index\n# `use_tantivy=True` by default\ntable.create_fts_index(\"text\", use_tantivy=False)\ntable.search(\"puppy\").limit(10).select([\"text\"]).to_list()\n# [{'text': 'Frodo was a happy puppy', '_score': 0.6931471824645996}]\n# ...\n";
+export const PyBasicFts = "uri = \"data/sample-lancedb\"\ndb = lancedb.connect(uri)\n\ntable = db.create_table(\n    \"my_table_fts\",\n    data=[\n        {\"vector\": [3.1, 4.1], \"text\": \"Frodo was a happy puppy\"},\n        {\"vector\": [5.9, 26.5], \"text\": \"There are several kittens playing\"},\n    ],\n    mode=\"overwrite\",\n)\n\ntable.create_fts_index(\"text\")\ntable.search(\"puppy\").limit(10).select([\"text\"]).to_list()\n# [{'text': 'Frodo was a happy puppy', '_score': 0.6931471824645996}]\n# ...\n";
 
 export const PyBasicFtsAsync = "uri = \"data/sample-lancedb\"\nasync_db = await lancedb.connect_async(uri)\n\nasync_tbl = await async_db.create_table(\n    \"my_table_fts_async\",\n    data=[\n        {\"vector\": [3.1, 4.1], \"text\": \"Frodo was a happy puppy\"},\n        {\"vector\": [5.9, 26.5], \"text\": \"There are several kittens playing\"},\n    ],\n    mode=\"overwrite\",\n)\n\n# async API uses our native FTS algorithm\nawait async_tbl.create_index(\"text\", config=FTS())\nawait (await async_tbl.search(\"puppy\")).select([\"text\"]).limit(10).to_list()\n# [{'text': 'Frodo was a happy puppy', '_score': 0.6931471824645996}]\n# ...\n";
 
@@ -24,7 +24,7 @@ export const PyExhaustiveSearchAsyncCosine = "(\n    await (await async_tbl.sear
 
 export const PyExhaustiveSearchCosine = "tbl.search(np.random.random((1536))).distance_type(\"cosine\").limit(10).to_list()\n";
 
-export const PyFtsConfigFolding = "table.create_fts_index(\n    \"text\",\n    use_tantivy=False,\n    language=\"French\",\n    stem=True,\n    ascii_folding=True,\n    replace=True,\n)\n";
+export const PyFtsConfigFolding = "table.create_fts_index(\n    \"text\",\n    language=\"French\",\n    stem=True,\n    ascii_folding=True,\n    replace=True,\n)\n";
 
 export const PyFtsConfigFoldingAsync = "await async_tbl.create_index(\n    \"text\", config=FTS(language=\"French\", stem=True, ascii_folding=True)\n)\n";
 
@@ -44,7 +44,7 @@ export const PyFtsPrefiltering = "table.search(\"puppy\").limit(10).where(\"text
 
 export const PyFtsPrefilteringAsync = "await (await async_tbl.search(\"puppy\")).limit(10).where(\"text='foo'\").to_list()\n";
 
-export const PyFtsWithPosition = "table.create_fts_index(\"text\", use_tantivy=False, with_position=True, replace=True)\n";
+export const PyFtsWithPosition = "table.create_fts_index(\"text\", with_position=True, replace=True)\n";
 
 export const PyFtsWithPositionAsync = "await async_tbl.create_index(\"text\", config=FTS(with_position=True))\n";
 

tests/py/test_integrations.py

@@ -1431,7 +1431,6 @@ def extract_video_id(youtube_url: str) -> str:
             uri="./tmp/lancedb",
             table_name="youtube_transcripts",
             search_type=SearchType.hybrid,
-            use_tantivy=False,
             embedder=OpenAIEmbedder(id="text-embedding-3-small"),
         ),
     )

tests/py/test_search.py

@@ -181,7 +181,7 @@ def test_fts_fuzzy_query():
         ),
         mode="overwrite",
     )
-    table.create_fts_index("text", use_tantivy=False, replace=True)
+    table.create_fts_index("text", replace=True)
 
     results = table.search(MatchQuery("foo", "text", fuzziness=1)).to_pandas()
     assert len(results) == 4
@@ -219,7 +219,7 @@ def test_fts_boost_query():
         ),
         mode="overwrite",
     )
-    table.create_fts_index("desc", use_tantivy=False, replace=True)
+    table.create_fts_index("desc", replace=True)
 
     results = table.search(
         BoostQuery(
@@ -252,9 +252,7 @@ def test_fts_native():
         mode="overwrite",
     )
 
-    # passing `use_tantivy=False` to use lance FTS index
-    # `use_tantivy=True` by default
-    table.create_fts_index("text", use_tantivy=False)
+    table.create_fts_index("text")
     table.search("puppy").limit(10).select(["text"]).to_list()
     # [{'text': 'Frodo was a happy puppy', '_score': 0.6931471824645996}]
     # ...
@@ -265,7 +263,6 @@ def test_fts_native():
     # --8<-- [start:fts_config_folding]
     table.create_fts_index(
         "text",
-        use_tantivy=False,
         language="French",
         stem=True,
         ascii_folding=True,
@@ -279,7 +276,7 @@ def test_fts_native():
     table.search("puppy").limit(10).where("text='foo'", prefilter=False).to_list()
     # --8<-- [end:fts_postfiltering]
     # --8<-- [start:fts_with_position]
-    table.create_fts_index("text", use_tantivy=False, with_position=True, replace=True)
+    table.create_fts_index("text", with_position=True, replace=True)
     # --8<-- [end:fts_with_position]
     # --8<-- [start:fts_incremental_index]
     table.add([{"vector": [3.1, 4.1], "text": "Frodo was a happy puppy"}])