diff --git a/CHANGELOG.md b/CHANGELOG.md index fa8ca96..21bfbf1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file. The format ## Unreleased +### Changed + +- **Breaking:** the bm25-named API has been renamed to paradedb, and indexes always use the `paradedb` index access method (requires pg_search 0.25.0+). `BM25Field` is now `ParadeDBField`, `validate_bm25_index` is now `validate_paradedb_index`, the Alembic operations `op.create_bm25_index`/`op.drop_bm25_index`/`op.reindex_bm25` are now `op.create_paradedb_index`/`op.drop_paradedb_index`/`op.reindex_paradedb` (with operation classes `CreateParadeDBIndexOp`/`DropParadeDBIndexOp`/`ReindexParadeDBOp`), and the errors `BM25ValidationError`/`InvalidBM25FieldError` are now `ParadeDBValidationError`/`InvalidParadeDBFieldError`. `Index(postgresql_using="paradedb")` is the only recognized access method; index DDL always emits `USING paradedb`. Existing migrations that call the bm25-named operations must be updated to the paradedb names. + ## [0.8.0] - 2026-07-14 ### Changed diff --git a/README.md b/README.md index d4480e3..fc3eb69 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ ## ParadeDB for SQLAlchemy -The official [SQLAlchemy](https://www.sqlalchemy.org/) integration for [ParadeDB](https://paradedb.com) (powered by the [`pg_search`](https://github.com/paradedb/paradedb) Postgres extension), including first-class support for managing BM25 indexes with Alembic and running queries using the full ParadeDB API. Follow the [getting started guide](https://docs.paradedb.com/documentation/getting-started/environment#sqlalchemy) to begin. +The official [SQLAlchemy](https://www.sqlalchemy.org/) integration for [ParadeDB](https://paradedb.com) (powered by the [`pg_search`](https://github.com/paradedb/paradedb) Postgres extension), including first-class support for managing ParadeDB indexes with Alembic and running queries using the full ParadeDB API. Follow the [getting started guide](https://docs.paradedb.com/documentation/getting-started/environment#sqlalchemy) to begin. ## Requirements & Compatibility @@ -44,7 +44,7 @@ The official [SQLAlchemy](https://www.sqlalchemy.org/) integration for [ParadeDB | ---------- | ----------------------------- | | Python | 3.10+ | | SQLAlchemy | 2.0.32+ | -| ParadeDB | 0.22.0+ | +| ParadeDB | 0.25.0+ | | PostgreSQL | 15+ (with ParadeDB extension) | ## Examples diff --git a/examples/autocomplete/setup.py b/examples/autocomplete/setup.py index a948a0d..6b36b7f 100644 --- a/examples/autocomplete/setup.py +++ b/examples/autocomplete/setup.py @@ -34,15 +34,15 @@ class Product(Base): Index( "products_autocomplete_bm25_idx", - indexing.BM25Field(Product.id), - indexing.BM25Field(Product.description), - indexing.BM25Field( + indexing.ParadeDBField(Product.id), + indexing.ParadeDBField(Product.description), + indexing.ParadeDBField( Product.description, tokenizer=tokenizer.ngram(3, 8, options={"prefix_only": True, "alias": "description_ngram"}), ), - indexing.BM25Field(Product.category, tokenizer=tokenizer.literal()), - indexing.BM25Field(Product.rating), - postgresql_using="bm25", + indexing.ParadeDBField(Product.category, tokenizer=tokenizer.literal()), + indexing.ParadeDBField(Product.rating), + postgresql_using="paradedb", postgresql_with={"key_field": "id"}, ) diff --git a/examples/faceted_search/setup.py b/examples/faceted_search/setup.py index e640021..46e9b30 100644 --- a/examples/faceted_search/setup.py +++ b/examples/faceted_search/setup.py @@ -34,11 +34,11 @@ class Product(Base): Index( "products_facets_bm25_idx", - indexing.BM25Field(Product.id), - indexing.BM25Field(Product.description), - indexing.BM25Field(Product.category, tokenizer=tokenizer.literal()), - indexing.BM25Field(Product.rating), - postgresql_using="bm25", + indexing.ParadeDBField(Product.id), + indexing.ParadeDBField(Product.description), + indexing.ParadeDBField(Product.category, tokenizer=tokenizer.literal()), + indexing.ParadeDBField(Product.rating), + postgresql_using="paradedb", postgresql_with={"key_field": "id"}, ) diff --git a/examples/hybrid_rrf/setup.py b/examples/hybrid_rrf/setup.py index a8fb481..12679dd 100644 --- a/examples/hybrid_rrf/setup.py +++ b/examples/hybrid_rrf/setup.py @@ -34,11 +34,11 @@ class Product(Base): Index( "products_hybrid_rrf_bm25_idx", - indexing.BM25Field(Product.id), - indexing.BM25Field(Product.description), - indexing.BM25Field(Product.category, tokenizer=tokenizer.literal()), - indexing.BM25Field(Product.rating), - postgresql_using="bm25", + indexing.ParadeDBField(Product.id), + indexing.ParadeDBField(Product.description), + indexing.ParadeDBField(Product.category, tokenizer=tokenizer.literal()), + indexing.ParadeDBField(Product.rating), + postgresql_using="paradedb", postgresql_with={"key_field": "id"}, ) diff --git a/examples/more_like_this/setup.py b/examples/more_like_this/setup.py index df41f0b..fc45452 100644 --- a/examples/more_like_this/setup.py +++ b/examples/more_like_this/setup.py @@ -34,11 +34,11 @@ class Product(Base): Index( "products_mlt_bm25_idx", - indexing.BM25Field(Product.id), - indexing.BM25Field(Product.description), - indexing.BM25Field(Product.category, tokenizer=tokenizer.literal()), - indexing.BM25Field(Product.rating), - postgresql_using="bm25", + indexing.ParadeDBField(Product.id), + indexing.ParadeDBField(Product.description), + indexing.ParadeDBField(Product.category, tokenizer=tokenizer.literal()), + indexing.ParadeDBField(Product.rating), + postgresql_using="paradedb", postgresql_with={"key_field": "id"}, ) diff --git a/examples/quickstart/setup.py b/examples/quickstart/setup.py index 988ace0..2b5eda2 100644 --- a/examples/quickstart/setup.py +++ b/examples/quickstart/setup.py @@ -34,11 +34,11 @@ class Product(Base): Index( "products_bm25_idx", - indexing.BM25Field(Product.id), - indexing.BM25Field(Product.description), - indexing.BM25Field(Product.category, tokenizer=tokenizer.literal()), - indexing.BM25Field(Product.rating), - postgresql_using="bm25", + indexing.ParadeDBField(Product.id), + indexing.ParadeDBField(Product.description), + indexing.ParadeDBField(Product.category, tokenizer=tokenizer.literal()), + indexing.ParadeDBField(Product.rating), + postgresql_using="paradedb", postgresql_with={"key_field": "id"}, ) diff --git a/examples/rag/setup.py b/examples/rag/setup.py index b2086d4..a5f9bd5 100644 --- a/examples/rag/setup.py +++ b/examples/rag/setup.py @@ -29,9 +29,9 @@ class Document(Base): Index( "documents_bm25_idx", - indexing.BM25Field(Document.id), - indexing.BM25Field(Document.content), - postgresql_using="bm25", + indexing.ParadeDBField(Document.id), + indexing.ParadeDBField(Document.content), + postgresql_using="paradedb", postgresql_with={"key_field": "id"}, ) diff --git a/paradedb/__init__.py b/paradedb/__init__.py index 4757084..98f8ca5 100644 --- a/paradedb/__init__.py +++ b/paradedb/__init__.py @@ -6,7 +6,7 @@ paradedb_verify_index, ) from .sqlalchemy.facets import with_rows -from .sqlalchemy.indexing import BM25Field, assert_indexed, describe +from .sqlalchemy.indexing import ParadeDBField, assert_indexed, describe from .sqlalchemy.tokenizer import Tokenizer from .sqlalchemy import tokenizer from .sqlalchemy.pdb import agg, alias, score, snippet, snippet_positions, snippets @@ -31,7 +31,7 @@ ) __all__ = [ - "BM25Field", + "ParadeDBField", "ProximityExpr", "Tokenizer", "agg", diff --git a/paradedb/sqlalchemy/alembic.py b/paradedb/sqlalchemy/alembic.py index f909fed..89eaf74 100644 --- a/paradedb/sqlalchemy/alembic.py +++ b/paradedb/sqlalchemy/alembic.py @@ -24,8 +24,8 @@ def _quote_qualified(schema: str | None, name: str) -> str: return _quote_ident(name) -@Operations.register_operation("create_bm25_index") -class CreateBM25IndexOp(MigrateOperation): +@Operations.register_operation("create_paradedb_index") +class CreateParadeDBIndexOp(MigrateOperation): def __init__( self, index_name: str, @@ -44,7 +44,7 @@ def __init__( self.where = where @classmethod - def create_bm25_index( + def create_paradedb_index( cls, operations: Operations, index_name: str, @@ -67,24 +67,24 @@ def create_bm25_index( ) def reverse(self) -> MigrateOperation: - return DropBM25IndexOp(index_name=self.index_name, if_exists=True, schema=self.table_schema) + return DropParadeDBIndexOp(index_name=self.index_name, if_exists=True, schema=self.table_schema) -@Operations.implementation_for(CreateBM25IndexOp) -def _create_bm25_index_impl(operations: Operations, operation: CreateBM25IndexOp) -> None: +@Operations.implementation_for(CreateParadeDBIndexOp) +def _create_paradedb_index_impl(operations: Operations, operation: CreateParadeDBIndexOp) -> None: expressions_sql = ", ".join(operation.expressions) sql = ( f"CREATE INDEX {_quote_ident(operation.index_name)} " f"ON {_quote_qualified(operation.table_schema, operation.table_name)} " - f"USING bm25 ({expressions_sql}) WITH (key_field={_quote_literal(operation.key_field)})" + f"USING paradedb ({expressions_sql}) WITH (key_field={_quote_literal(operation.key_field)})" ) if operation.where is not None: sql += f" WHERE {operation.where}" operations.execute(sql) -@renderers.dispatch_for(CreateBM25IndexOp) -def _render_create_bm25_index_op(autogen_context, op: CreateBM25IndexOp) -> str: +@renderers.dispatch_for(CreateParadeDBIndexOp) +def _render_create_paradedb_index_op(autogen_context, op: CreateParadeDBIndexOp) -> str: parts = [ repr(op.index_name), repr(op.table_name), @@ -95,11 +95,11 @@ def _render_create_bm25_index_op(autogen_context, op: CreateBM25IndexOp) -> str: parts.append(f"table_schema={op.table_schema!r}") if op.where is not None: parts.append(f"where={op.where!r}") - return f"op.create_bm25_index({', '.join(parts)})" + return f"op.create_paradedb_index({', '.join(parts)})" -@Operations.register_operation("drop_bm25_index") -class DropBM25IndexOp(MigrateOperation): +@Operations.register_operation("drop_paradedb_index") +class DropParadeDBIndexOp(MigrateOperation): def __init__( self, index_name: str, @@ -120,7 +120,7 @@ def __init__( self.where = where @classmethod - def drop_bm25_index( + def drop_paradedb_index( cls, operations: Operations, index_name: str, @@ -146,9 +146,9 @@ def drop_bm25_index( def reverse(self) -> MigrateOperation: if self.table_name is None or self.expressions is None or self.key_field is None: - raise NotImplementedError("DropBM25IndexOp requires recreate metadata for Alembic downgrade generation") + raise NotImplementedError("DropParadeDBIndexOp requires recreate metadata for Alembic downgrade generation") - return CreateBM25IndexOp( + return CreateParadeDBIndexOp( index_name=self.index_name, table_name=self.table_name, expressions=self.expressions, @@ -158,14 +158,14 @@ def reverse(self) -> MigrateOperation: ) -@Operations.implementation_for(DropBM25IndexOp) -def _drop_bm25_index_impl(operations: Operations, operation: DropBM25IndexOp) -> None: +@Operations.implementation_for(DropParadeDBIndexOp) +def _drop_paradedb_index_impl(operations: Operations, operation: DropParadeDBIndexOp) -> None: if_exists_sql = " IF EXISTS" if operation.if_exists else "" operations.execute(f"DROP INDEX{if_exists_sql} {_quote_qualified(operation.schema, operation.index_name)}") -@renderers.dispatch_for(DropBM25IndexOp) -def _render_drop_bm25_index_op(autogen_context, op: DropBM25IndexOp) -> str: +@renderers.dispatch_for(DropParadeDBIndexOp) +def _render_drop_paradedb_index_op(autogen_context, op: DropParadeDBIndexOp) -> str: parts = [repr(op.index_name), f"if_exists={op.if_exists!r}"] if op.schema is not None: parts.append(f"schema={op.schema!r}") @@ -177,45 +177,45 @@ def _render_drop_bm25_index_op(autogen_context, op: DropBM25IndexOp) -> str: parts.append(f"key_field={op.key_field!r}") if op.where is not None: parts.append(f"where={op.where!r}") - return f"op.drop_bm25_index({', '.join(parts)})" + return f"op.drop_paradedb_index({', '.join(parts)})" -@Operations.register_operation("reindex_bm25") -class ReindexBM25Op(MigrateOperation): +@Operations.register_operation("reindex_paradedb") +class ReindexParadeDBOp(MigrateOperation): def __init__(self, index_name: str, concurrently: bool = False, schema: str | None = None) -> None: self.index_name = index_name self.concurrently = concurrently self.schema = schema @classmethod - def reindex_bm25( + def reindex_paradedb( cls, operations: Operations, index_name: str, concurrently: bool = False, schema: str | None = None ) -> MigrateOperation: return operations.invoke(cls(index_name=index_name, concurrently=concurrently, schema=schema)) -@Operations.implementation_for(ReindexBM25Op) -def _reindex_bm25_impl(operations: Operations, operation: ReindexBM25Op) -> None: +@Operations.implementation_for(ReindexParadeDBOp) +def _reindex_paradedb_impl(operations: Operations, operation: ReindexParadeDBOp) -> None: concurrently_sql = " CONCURRENTLY" if operation.concurrently else "" operations.execute(f"REINDEX INDEX{concurrently_sql} {_quote_qualified(operation.schema, operation.index_name)}") -@renderers.dispatch_for(ReindexBM25Op) -def _render_reindex_bm25_op(autogen_context, op: ReindexBM25Op) -> str: +@renderers.dispatch_for(ReindexParadeDBOp) +def _render_reindex_paradedb_op(autogen_context, op: ReindexParadeDBOp) -> str: parts = [repr(op.index_name), f"concurrently={op.concurrently!r}"] if op.schema is not None: parts.append(f"schema={op.schema!r}") - return f"op.reindex_bm25({', '.join(parts)})" + return f"op.reindex_paradedb({', '.join(parts)})" # --------------------------------------------------------------------------- # Autogenerate comparator # --------------------------------------------------------------------------- -def _autogen_bm25_meta_indexes( +def _autogen_paradedb_meta_indexes( metadata, effective_schemas: set[str], *, default_schema: str ) -> dict[tuple[str, str], object]: - """Return {(schema, index_name): Index} for all BM25 indexes in MetaData.""" - from .indexing import _is_bm25_index + """Return {(schema, index_name): Index} for all ParadeDB indexes in MetaData.""" + from .indexing import _is_paradedb_index result: dict[tuple[str, str], object] = {} for table in metadata.tables.values(): @@ -223,23 +223,23 @@ def _autogen_bm25_meta_indexes( if schema not in effective_schemas: continue for index in table.indexes: - if _is_bm25_index(index): + if _is_paradedb_index(index): result[(schema, index.name)] = index return result -def _autogen_bm25_db_indexes(conn, effective_schemas: set[str]) -> dict[tuple[str, str], dict]: +def _autogen_paradedb_db_indexes(conn, effective_schemas: set[str]) -> dict[tuple[str, str], dict]: """Return {(schema, index_name): {table_name, expressions, key_field, where}} from pg_indexes.""" from .indexing import ( _extract_key_field, _extract_where_clause, - _introspect_bm25_index_rows, + _introspect_paradedb_index_rows, _normalize_reloption_value, ) result: dict[tuple[str, str], dict] = {} for schema in effective_schemas: - rows = _introspect_bm25_index_rows(conn, schema_name=schema) + rows = _introspect_paradedb_index_rows(conn, schema_name=schema) for row in rows: key = (row["schemaname"], row["indexname"]) index_entry = result.setdefault( @@ -257,7 +257,7 @@ def _autogen_bm25_db_indexes(conn, effective_schemas: set[str]) -> dict[tuple[st return result -def _render_bm25_expression(expr: ClauseElement) -> str: +def _render_paradedb_expression(expr: ClauseElement) -> str: return str(expr.compile(dialect=postgresql.dialect(), compile_kwargs={"literal_binds": True})) # type: ignore[no-untyped-call] @@ -285,8 +285,8 @@ def _strip_match(match: re.Match[str]) -> str: return qualifier_re.sub(_strip_match, expr) -def _normalize_bm25_expression(expr: str) -> str: - """Normalize BM25 expression text to reduce false-positive autogen churn.""" +def _normalize_paradedb_expression(expr: str) -> str: + """Normalize ParadeDB expression text to reduce false-positive autogen churn.""" normalized = "".join(expr.split()) normalized = normalized.replace('"', "") normalized = normalized.replace("::text", "") @@ -343,7 +343,7 @@ def _strip_non_pdb_qualifiers(expr: str) -> str: def _normalized_expression_list(expressions: list[str]) -> list[str]: - return [_normalize_bm25_expression(expr) for expr in expressions] + return [_normalize_paradedb_expression(expr) for expr in expressions] def _normalize_where(clause: str | None) -> str | None: @@ -390,15 +390,15 @@ def _render_where_from_index(index) -> str | None: return _strip_relation_qualifiers(str(where_clause), index.table.name, index.table.schema) -def _suppress_standard_bm25_ops(upgrade_ops, bm25_names: set[str]) -> None: - """Remove any standard Alembic CreateIndexOp/DropIndexOp for BM25 indexes.""" +def _suppress_standard_paradedb_ops(upgrade_ops, paradedb_names: set[str]) -> None: + """Remove any standard Alembic CreateIndexOp/DropIndexOp for ParadeDB indexes.""" from alembic.operations.ops import CreateIndexOp, DropIndexOp, ModifyTableOps # Filter top-level (rare, but defensive) upgrade_ops.ops[:] = [ op for op in upgrade_ops.ops - if not (isinstance(op, (CreateIndexOp, DropIndexOp)) and op.index_name in bm25_names) + if not (isinstance(op, (CreateIndexOp, DropIndexOp)) and op.index_name in paradedb_names) ] # Filter inside ModifyTableOps (the normal location for index ops) for op in upgrade_ops.ops: @@ -406,13 +406,13 @@ def _suppress_standard_bm25_ops(upgrade_ops, bm25_names: set[str]) -> None: op.ops[:] = [ sub_op for sub_op in op.ops - if not (isinstance(sub_op, (CreateIndexOp, DropIndexOp)) and sub_op.index_name in bm25_names) + if not (isinstance(sub_op, (CreateIndexOp, DropIndexOp)) and sub_op.index_name in paradedb_names) ] @comparators.dispatch_for("schema", priority=DispatchPriority.LAST) -def _compare_bm25_indexes(autogen_context, upgrade_ops, schemas) -> PriorityDispatchResult: - """Autogenerate comparator: emit BM25 create/drop ops and suppress incorrect standard ops.""" +def _compare_paradedb_indexes(autogen_context, upgrade_ops, schemas) -> PriorityDispatchResult: + """Autogenerate comparator: emit ParadeDB create/drop ops and suppress incorrect standard ops.""" conn = autogen_context.connection metadata = autogen_context.metadata @@ -422,24 +422,24 @@ def _compare_bm25_indexes(autogen_context, upgrade_ops, schemas) -> PriorityDisp default_schema: str = conn.dialect.default_schema_name or "public" effective_schemas = {s if s is not None else default_schema for s in schemas} - db_bm25 = _autogen_bm25_db_indexes(conn, effective_schemas) - meta_bm25 = _autogen_bm25_meta_indexes(metadata, effective_schemas, default_schema=default_schema) + db_paradedb = _autogen_paradedb_db_indexes(conn, effective_schemas) + meta_paradedb = _autogen_paradedb_meta_indexes(metadata, effective_schemas, default_schema=default_schema) - all_bm25_names = {k[1] for k in db_bm25} | {k[1] for k in meta_bm25} - if not all_bm25_names: + all_paradedb_names = {k[1] for k in db_paradedb} | {k[1] for k in meta_paradedb} + if not all_paradedb_names: return PriorityDispatchResult.CONTINUE - # Remove any standard CreateIndexOp/DropIndexOp for BM25 indexes since - # those would render incorrect DDL (BM25Field expressions can't be + # Remove any standard CreateIndexOp/DropIndexOp for ParadeDB indexes since + # those would render incorrect DDL (ParadeDBField expressions can't be # round-tripped through the standard Inspector → Python code path). - _suppress_standard_bm25_ops(upgrade_ops, all_bm25_names) + _suppress_standard_paradedb_ops(upgrade_ops, all_paradedb_names) # Emit drop ops for indexes present in DB but absent from MetaData. - for key in db_bm25: - if key not in meta_bm25: - db = db_bm25[key] + for key in db_paradedb: + if key not in meta_paradedb: + db = db_paradedb[key] upgrade_ops.ops.append( - DropBM25IndexOp( + DropParadeDBIndexOp( index_name=key[1], if_exists=True, schema=key[0], @@ -452,15 +452,15 @@ def _compare_bm25_indexes(autogen_context, upgrade_ops, schemas) -> PriorityDisp # Emit create ops for indexes present in MetaData but absent from DB. # Also re-create indexes whose expression list, key_field, or WHERE clause differs from the DB. - for key, index in meta_bm25.items(): + for key, index in meta_paradedb.items(): with_opts = index.dialect_options["postgresql"].get("with") or {} key_field = with_opts.get("key_field", "") expressions = [ - _strip_relation_qualifiers(_render_bm25_expression(expr), index.table.name, index.table.schema) + _strip_relation_qualifiers(_render_paradedb_expression(expr), index.table.name, index.table.schema) for expr in index.expressions ] meta_where = _render_where_from_index(index) - create_op = CreateBM25IndexOp( + create_op = CreateParadeDBIndexOp( index_name=index.name, table_name=index.table.name, expressions=expressions, @@ -469,10 +469,10 @@ def _compare_bm25_indexes(autogen_context, upgrade_ops, schemas) -> PriorityDisp where=meta_where, ) - if key not in db_bm25: + if key not in db_paradedb: upgrade_ops.ops.append(create_op) else: - db = db_bm25[key] + db = db_paradedb[key] expressions_changed = _normalized_expression_list(db["expressions"]) != _normalized_expression_list( expressions ) @@ -480,7 +480,7 @@ def _compare_bm25_indexes(autogen_context, upgrade_ops, schemas) -> PriorityDisp where_changed = _normalize_where(db.get("where")) != _normalize_where(meta_where) if expressions_changed or key_field_changed or where_changed: upgrade_ops.ops.append( - DropBM25IndexOp( + DropParadeDBIndexOp( index_name=key[1], if_exists=True, schema=key[0], diff --git a/paradedb/sqlalchemy/diagnostics.py b/paradedb/sqlalchemy/diagnostics.py index dd3de22..21ef573 100644 --- a/paradedb/sqlalchemy/diagnostics.py +++ b/paradedb/sqlalchemy/diagnostics.py @@ -14,13 +14,13 @@ def _exec_and_collect(conn, sql: str, params: list[Any]) -> list[dict[str, Any]] def paradedb_indexes(engine: Engine) -> list[dict[str, Any]]: - """Return metadata for all BM25 indexes from ``pdb.indexes()``.""" + """Return metadata for all ParadeDB indexes from ``pdb.indexes()``.""" with engine.connect() as conn: return _exec_and_collect(conn, "SELECT * FROM pdb.indexes()", []) def paradedb_index_segments(engine: Engine, index: str) -> list[dict[str, Any]]: - """Return segment metadata for a BM25 index from ``pdb.index_segments()``.""" + """Return segment metadata for a ParadeDB index from ``pdb.index_segments()``.""" with engine.connect() as conn: return _exec_and_collect(conn, "SELECT * FROM pdb.index_segments(%s::regclass)", [index]) @@ -36,7 +36,7 @@ def paradedb_verify_index( on_error_stop: bool = False, segment_ids: Sequence[int] | None = None, ) -> list[dict[str, Any]]: - """Run ``pdb.verify_index()`` for one BM25 index.""" + """Run ``pdb.verify_index()`` for one ParadeDB index.""" sql_parts = ["SELECT * FROM pdb.verify_index(%s::regclass"] params: list[Any] = [index] if heapallindexed: @@ -72,7 +72,7 @@ def paradedb_verify_all_indexes( report_progress: bool = False, on_error_stop: bool = False, ) -> list[dict[str, Any]]: - """Run ``pdb.verify_all_indexes()`` across BM25 indexes.""" + """Run ``pdb.verify_all_indexes()`` across ParadeDB indexes.""" named_params: list[tuple[str, str, Any]] = [] if schema_pattern is not None: named_params.append(("schema_pattern", "text", schema_pattern)) diff --git a/paradedb/sqlalchemy/errors.py b/paradedb/sqlalchemy/errors.py index 75de3da..ad1ebef 100644 --- a/paradedb/sqlalchemy/errors.py +++ b/paradedb/sqlalchemy/errors.py @@ -13,24 +13,24 @@ class InvalidMoreLikeThisOptionsError(InvalidArgumentError): """Raised when more_like_this options are missing/conflicting/out-of-range.""" -class BM25ValidationError(ParadeDBError, ValueError): - """Base class for BM25 index validation errors.""" +class ParadeDBValidationError(ParadeDBError, ValueError): + """Base class for ParadeDB index validation errors.""" -class MissingKeyFieldError(BM25ValidationError): - """Raised when a BM25 index is missing key_field option.""" +class MissingKeyFieldError(ParadeDBValidationError): + """Raised when a ParadeDB index is missing key_field option.""" -class InvalidKeyFieldError(BM25ValidationError): - """Raised when BM25 key_field is not part of index fields.""" +class InvalidKeyFieldError(ParadeDBValidationError): + """Raised when ParadeDB key_field is not part of index fields.""" -class DuplicateTokenizerAliasError(BM25ValidationError): - """Raised when tokenizer aliases are duplicated in one BM25 index.""" +class DuplicateTokenizerAliasError(ParadeDBValidationError): + """Raised when tokenizer aliases are duplicated in one ParadeDB index.""" -class InvalidBM25FieldError(BM25ValidationError): - """Raised when non-BM25Field expressions are used in a BM25 index.""" +class InvalidParadeDBFieldError(ParadeDBValidationError): + """Raised when non-ParadeDBField expressions are used in a ParadeDB index.""" class RuntimeGuardError(ParadeDBError, ValueError): @@ -58,4 +58,4 @@ class FacetRequiresParadeDBPredicateError(FacetRuntimeError): class FieldNotIndexedError(ParadeDBError): - """Raised when a column is not covered by any BM25 index on its table.""" + """Raised when a column is not covered by any ParadeDB index on its table.""" diff --git a/paradedb/sqlalchemy/indexing.py b/paradedb/sqlalchemy/indexing.py index 90cde0b..c10fca3 100644 --- a/paradedb/sqlalchemy/indexing.py +++ b/paradedb/sqlalchemy/indexing.py @@ -19,14 +19,14 @@ DuplicateTokenizerAliasError, FieldNotIndexedError, InvalidArgumentError, - InvalidBM25FieldError, InvalidKeyFieldError, + InvalidParadeDBFieldError, MissingKeyFieldError, ) -class BM25Field(ColumnElement[Any]): - """Represents a ParadeDB BM25 index field expression.""" +class ParadeDBField(ColumnElement[Any]): + """Represents a ParadeDB index field expression.""" inherit_cache = True _traverse_internals = [ @@ -43,43 +43,43 @@ def table(self): # pragma: no cover - SQLAlchemy internals may use this dynamic return getattr(self.expr, "table", None) -@compiles(BM25Field, "postgresql") -def _compile_bm25_field(element: BM25Field, compiler, **kw: Any) -> str: +@compiles(ParadeDBField, "postgresql") +def _compile_paradedb_field(element: ParadeDBField, compiler, **kw: Any) -> str: expr_sql = compiler.process(element.expr, **kw) if element.tokenizer is None: - if isinstance(element.expr, PDBCast) or _bm25_field_name(element) is None: + if isinstance(element.expr, PDBCast) or _paradedb_field_name(element) is None: return f"({expr_sql})" return expr_sql return f"(({expr_sql})::{element.tokenizer.render()})" -@compiles(BM25Field) -def _compile_bm25_field_default(element: BM25Field, compiler, **kw: Any) -> str: - raise CompileError("BM25Field is only supported for PostgreSQL dialects") +@compiles(ParadeDBField) +def _compile_paradedb_field_default(element: ParadeDBField, compiler, **kw: Any) -> str: + raise CompileError("ParadeDBField is only supported for PostgreSQL dialects") -def _is_bm25_index(index: Index) -> bool: +def _is_paradedb_index(index: Index) -> bool: using = index.dialect_options["postgresql"].get("using") - return bool(using and str(using).lower() == "bm25") + return bool(using) and str(using).lower() == "paradedb" -def _bm25_field_name(field: BM25Field) -> str | None: +def _paradedb_field_name(field: ParadeDBField) -> str | None: return getattr(getattr(field, "expr", None), "name", None) -def validate_bm25_index(index: Index) -> None: - if not _is_bm25_index(index): +def validate_paradedb_index(index: Index) -> None: + if not _is_paradedb_index(index): return if not index.expressions: - raise InvalidBM25FieldError("BM25 indexes must include at least one BM25Field") + raise InvalidParadeDBFieldError("ParadeDB indexes must include at least one ParadeDBField") - if not all(isinstance(expr, BM25Field) for expr in index.expressions): - raise InvalidBM25FieldError("BM25 indexes must use BM25Field for every indexed field") + if not all(isinstance(expr, ParadeDBField) for expr in index.expressions): + raise InvalidParadeDBFieldError("ParadeDB indexes must use ParadeDBField for every indexed field") aliases: set[str] = set() for expr in index.expressions: - if not isinstance(expr, BM25Field): + if not isinstance(expr, ParadeDBField): continue tokenizer = expr.tokenizer if tokenizer is None: @@ -90,31 +90,31 @@ def validate_bm25_index(index: Index) -> None: continue if alias in aliases: - raise DuplicateTokenizerAliasError(f"Duplicate tokenizer alias '{alias}' in BM25 index") + raise DuplicateTokenizerAliasError(f"Duplicate tokenizer alias '{alias}' in ParadeDB index") aliases.add(alias) with_options = index.dialect_options["postgresql"].get("with") or {} key_field = with_options.get("key_field") if not key_field: - raise MissingKeyFieldError("BM25 indexes require postgresql_with={'key_field': ''}") + raise MissingKeyFieldError("ParadeDB indexes require postgresql_with={'key_field': ''}") - field_names = {_bm25_field_name(expr) for expr in index.expressions if isinstance(expr, BM25Field)} + field_names = {_paradedb_field_name(expr) for expr in index.expressions if isinstance(expr, ParadeDBField)} if key_field not in field_names: - raise InvalidKeyFieldError(f"BM25 key_field '{key_field}' must match one of the indexed BM25Field columns") + raise InvalidKeyFieldError(f"key_field '{key_field}' must match one of the indexed ParadeDBField columns") first_field = index.expressions[0] - if not isinstance(first_field, BM25Field): - raise InvalidBM25FieldError("BM25 indexes must use BM25Field for every indexed field") - first_field_name = _bm25_field_name(first_field) + if not isinstance(first_field, ParadeDBField): + raise InvalidParadeDBFieldError("ParadeDB indexes must use ParadeDBField for every indexed field") + first_field_name = _paradedb_field_name(first_field) if first_field_name != key_field: - raise InvalidKeyFieldError(f"BM25 key_field '{key_field}' must be the first indexed BM25Field") + raise InvalidKeyFieldError(f"key_field '{key_field}' must be the first indexed ParadeDBField") if first_field.tokenizer is not None: - raise InvalidKeyFieldError(f"BM25 key_field '{key_field}' must be untokenized") + raise InvalidKeyFieldError(f"key_field '{key_field}' must be untokenized") @event.listens_for(Index, "before_create") -def _validate_bm25_before_create(index: Index, connection, **kw: Any) -> None: - validate_bm25_index(index) +def _validate_paradedb_before_create(index: Index, connection, **kw: Any) -> None: + validate_paradedb_index(index) @dataclass(frozen=True) @@ -168,8 +168,8 @@ def _split_top_level_csv(expr: str) -> list[str]: return parts -def _extract_bm25_field_list(indexdef: str) -> list[str]: - marker = re.search(r"USING\s+bm25\s*\(", indexdef, re.IGNORECASE) +def _extract_paradedb_field_list(indexdef: str) -> list[str]: + marker = re.search(r"USING\s+(?:bm25|paradedb)\s*\(", indexdef, re.IGNORECASE) if marker is None: return [] @@ -299,7 +299,7 @@ def _normalize_reloption_value(value: str | None) -> str | None: return v -def _introspect_bm25_index_rows(conn, *, schema_name: str, table_name: str | None = None): +def _introspect_paradedb_index_rows(conn, *, schema_name: str, table_name: str | None = None): return ( conn.execute( text( @@ -308,6 +308,7 @@ def _introspect_bm25_index_rows(conn, *, schema_name: str, table_name: str | Non ns.nspname AS schemaname, tbl.relname AS tablename, idx.relname AS indexname, + am.amname AS amname, pg_get_indexdef(idx.oid) AS indexdef, split_part(opt.opt, '=', 2) AS key_field, key_ord.ord::int AS ordinality, @@ -317,6 +318,7 @@ def _introspect_bm25_index_rows(conn, *, schema_name: str, table_name: str | Non JOIN pg_namespace AS ns ON ns.oid = idx.relnamespace JOIN pg_index AS i ON i.indexrelid = idx.oid JOIN pg_class AS tbl ON tbl.oid = i.indrelid + JOIN pg_am AS am ON am.oid = idx.relam LEFT JOIN LATERAL ( SELECT opt FROM unnest(COALESCE(idx.reloptions, ARRAY[]::text[])) AS opt @@ -329,7 +331,7 @@ def _introspect_bm25_index_rows(conn, *, schema_name: str, table_name: str | Non AND attr.attnum = key_ord.attnum WHERE ns.nspname = :schema_name AND (CAST(:table_name AS text) IS NULL OR tbl.relname = CAST(:table_name AS text)) - AND pg_get_indexdef(idx.oid) ILIKE '%USING bm25%' + AND am.amname IN ('bm25', 'paradedb') ORDER BY idx.relname, key_ord.ord """ ), @@ -344,7 +346,7 @@ def describe(engine: Engine, table, *, schema: str | None = None) -> list[IndexM table_schema = schema if schema is not None else getattr(table, "schema", None) with engine.connect() as conn: effective_schema = table_schema if table_schema is not None else _current_schema_name(conn) - rows = _introspect_bm25_index_rows( + rows = _introspect_paradedb_index_rows( conn, schema_name=effective_schema, table_name=table.name, @@ -411,7 +413,7 @@ def assert_indexed( tokenizer: str | None = None, schema: str | None = None, ) -> None: - """Raise :exc:`FieldNotIndexedError` if *column* is not covered by any BM25 index. + """Raise :exc:`FieldNotIndexedError` if *column* is not covered by any ParadeDB index. Args: engine: SQLAlchemy engine connected to the ParadeDB database. @@ -441,7 +443,7 @@ def assert_indexed( if tokenizer in idx_meta.tokenizers.get(col_name, ()): return # field is indexed with the requested tokenizer - msg = f"'{col_name}' is not indexed in any BM25 index on '{table.name}'" + msg = f"'{col_name}' is not indexed in any ParadeDB index on '{table.name}'" if tokenizer: msg += f" with tokenizer '{tokenizer}'" raise FieldNotIndexedError(msg) @@ -467,7 +469,7 @@ def validate_pushdown(stmt: Any) -> list[str]: if whereclause is None: warnings.append("No WHERE clause found; query will perform a full table scan without ParadeDB") elif not _inspect.has_paradedb_predicate(whereclause): - warnings.append("No ParadeDB predicate found in WHERE clause; query will not use a BM25 index") + warnings.append("No ParadeDB predicate found in WHERE clause; query will not use a ParadeDB index") if has_order_by(stmt) and not has_limit(stmt): warnings.append("ORDER BY is present without LIMIT; Top K pushdown to ParadeDB requires both") diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 1876733..561825d 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -71,7 +71,7 @@ def engine(db_url: str) -> Iterator[Engine]: with engine.begin() as conn: conn.execute( text( - "CREATE INDEX products_bm25_idx ON products USING bm25 (id, description, category, rating) WITH (key_field='id')" + "CREATE INDEX products_bm25_idx ON products USING paradedb (id, description, category, rating) WITH (key_field='id')" ) ) yield engine @@ -157,7 +157,7 @@ def paradedb_ready(engine: Engine) -> None: conn.execute(text("CALL paradedb.create_bm25_test_table(schema_name => 'public', table_name => 'mock_items')")) conn.execute( text( - "CREATE INDEX mock_items_bm25_idx ON mock_items USING bm25 (" + "CREATE INDEX mock_items_bm25_idx ON mock_items USING paradedb (" "id, description, category, rating, in_stock" ") WITH (key_field='id')" ) diff --git a/tests/integration/test_alembic_commands_integration.py b/tests/integration/test_alembic_commands_integration.py index 6906e5e..512aa0f 100644 --- a/tests/integration/test_alembic_commands_integration.py +++ b/tests/integration/test_alembic_commands_integration.py @@ -96,17 +96,17 @@ def test_alembic_command_autogenerate_upgrade_and_downgrade_bm25_index(engine, a Index( "search_idx", - indexing.BM25Field(MockItem.id), - indexing.BM25Field(MockItem.description), - indexing.BM25Field(MockItem.category), - indexing.BM25Field(MockItem.rating), - indexing.BM25Field(MockItem.in_stock), - indexing.BM25Field(MockItem.metadata_), - indexing.BM25Field(MockItem.created_at), - indexing.BM25Field(MockItem.last_updated_date), - indexing.BM25Field(MockItem.latest_available_time), - indexing.BM25Field(MockItem.weight_range), - postgresql_using="bm25", + indexing.ParadeDBField(MockItem.id), + indexing.ParadeDBField(MockItem.description), + indexing.ParadeDBField(MockItem.category), + indexing.ParadeDBField(MockItem.rating), + indexing.ParadeDBField(MockItem.in_stock), + indexing.ParadeDBField(MockItem.metadata_), + indexing.ParadeDBField(MockItem.created_at), + indexing.ParadeDBField(MockItem.last_updated_date), + indexing.ParadeDBField(MockItem.latest_available_time), + indexing.ParadeDBField(MockItem.weight_range), + postgresql_using="paradedb", postgresql_with={"key_field": "id"}, ) @@ -121,6 +121,8 @@ def test_alembic_command_autogenerate_upgrade_and_downgrade_bm25_index(engine, a f"CALL paradedb.create_bm25_test_table(schema_name => '{_AUTOGEN_SCHEMA}', table_name => 'mock_items')" ) ) + # Newer pg_search versions add an embedding column the model does not map. + conn.execute(text(f'ALTER TABLE "{_AUTOGEN_SCHEMA}"."mock_items" DROP COLUMN IF EXISTS embedding')) command.revision(config, message="add bm25 index", autogenerate=True, rev_id="rev_static") _assert_generated_migration( @@ -146,13 +148,13 @@ def test_alembic_command_autogenerate_upgrade_and_downgrade_bm25_index(engine, a def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### - op.create_bm25_index('search_idx', 'mock_items', ['id', 'description', 'category', 'rating', 'in_stock', 'metadata', 'created_at', 'last_updated_date', 'latest_available_time', 'weight_range'], key_field='id', table_schema='alembic_cmd_static') + op.create_paradedb_index('search_idx', 'mock_items', ['id', 'description', 'category', 'rating', 'in_stock', 'metadata', 'created_at', 'last_updated_date', 'latest_available_time', 'weight_range'], key_field='id', table_schema='alembic_cmd_static') # ### end Alembic commands ### def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### - op.drop_bm25_index('search_idx', if_exists=True, schema='alembic_cmd_static') + op.drop_paradedb_index('search_idx', if_exists=True, schema='alembic_cmd_static') # ### end Alembic commands ### ''', ) @@ -172,17 +174,17 @@ def test_alembic_command_autogenerate_changed_bm25_index(engine, alembic_config_ Index( "search_idx", - indexing.BM25Field(MockItem.id), - indexing.BM25Field(MockItem.description), - indexing.BM25Field(MockItem.category), - indexing.BM25Field(MockItem.rating), - indexing.BM25Field(MockItem.in_stock), - indexing.BM25Field(MockItem.metadata_), - indexing.BM25Field(MockItem.created_at), - indexing.BM25Field(MockItem.last_updated_date), - indexing.BM25Field(MockItem.latest_available_time), - indexing.BM25Field(MockItem.weight_range), - postgresql_using="bm25", + indexing.ParadeDBField(MockItem.id), + indexing.ParadeDBField(MockItem.description), + indexing.ParadeDBField(MockItem.category), + indexing.ParadeDBField(MockItem.rating), + indexing.ParadeDBField(MockItem.in_stock), + indexing.ParadeDBField(MockItem.metadata_), + indexing.ParadeDBField(MockItem.created_at), + indexing.ParadeDBField(MockItem.last_updated_date), + indexing.ParadeDBField(MockItem.latest_available_time), + indexing.ParadeDBField(MockItem.weight_range), + postgresql_using="paradedb", postgresql_with={"key_field": "id"}, ) @@ -197,10 +199,11 @@ def test_alembic_command_autogenerate_changed_bm25_index(engine, alembic_config_ f"CALL paradedb.create_bm25_test_table(schema_name => '{_CHANGED_SCHEMA}', table_name => 'mock_items')" ) ) + conn.execute(text(f'ALTER TABLE "{_CHANGED_SCHEMA}"."mock_items" DROP COLUMN IF EXISTS embedding')) conn.execute( text( f'CREATE INDEX "search_idx" ON "{_CHANGED_SCHEMA}"."mock_items" ' - "USING bm25 (id, description) WITH (key_field='id')" + "USING paradedb (id, description) WITH (key_field='id')" ) ) @@ -232,15 +235,15 @@ def test_alembic_command_autogenerate_changed_bm25_index(engine, alembic_config_ def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### - op.drop_bm25_index('search_idx', if_exists=True, schema='alembic_cmd_changed_static', table_name='mock_items', expressions=['id', 'description'], key_field='id') - op.create_bm25_index('search_idx', 'mock_items', ['id', 'description', 'category', 'rating', 'in_stock', 'metadata', 'created_at', 'last_updated_date', 'latest_available_time', 'weight_range'], key_field='id', table_schema='alembic_cmd_changed_static') + op.drop_paradedb_index('search_idx', if_exists=True, schema='alembic_cmd_changed_static', table_name='mock_items', expressions=['id', 'description'], key_field='id') + op.create_paradedb_index('search_idx', 'mock_items', ['id', 'description', 'category', 'rating', 'in_stock', 'metadata', 'created_at', 'last_updated_date', 'latest_available_time', 'weight_range'], key_field='id', table_schema='alembic_cmd_changed_static') # ### end Alembic commands ### def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### - op.drop_bm25_index('search_idx', if_exists=True, schema='alembic_cmd_changed_static') - op.create_bm25_index('search_idx', 'mock_items', ['id', 'description'], key_field='id', table_schema='alembic_cmd_changed_static') + op.drop_paradedb_index('search_idx', if_exists=True, schema='alembic_cmd_changed_static') + op.create_paradedb_index('search_idx', 'mock_items', ['id', 'description'], key_field='id', table_schema='alembic_cmd_changed_static') # ### end Alembic commands ### ''', ) diff --git a/tests/integration/test_alembic_integration.py b/tests/integration/test_alembic_integration.py index de333cd..fa9e431 100644 --- a/tests/integration/test_alembic_integration.py +++ b/tests/integration/test_alembic_integration.py @@ -10,7 +10,7 @@ import paradedb.sqlalchemy.alembic as pdb_alembic # noqa: F401 Ensure op registration from conftest import PARADEDB_SCAN_PROVIDERS -from paradedb.sqlalchemy.indexing import BM25Field +from paradedb.sqlalchemy.indexing import ParadeDBField from paradedb.sqlalchemy import tokenizer @@ -31,7 +31,7 @@ def _run_comparator(engine, metadata, schemas=None): ctx.connection = conn ctx.metadata = metadata upgrade_ops = UpgradeOps([]) - pdb_alembic._compare_bm25_indexes(ctx, upgrade_ops, schemas) + pdb_alembic._compare_paradedb_indexes(ctx, upgrade_ops, schemas) return upgrade_ops @@ -48,7 +48,7 @@ def test_alembic_create_reindex_drop_with_quoted_identifiers(engine): ctx = MigrationContext.configure(conn) op = Operations(ctx) - op.create_bm25_index(index_name, table_name, ["id", "description"], key_field="id") + op.create_paradedb_index(index_name, table_name, ["id", "description"], key_field="id") exists = conn.execute( text( @@ -63,8 +63,8 @@ def test_alembic_create_reindex_drop_with_quoted_identifiers(engine): ).scalar_one() assert exists == 1 - op.reindex_bm25(index_name) - op.drop_bm25_index(index_name, if_exists=True) + op.reindex_paradedb(index_name) + op.drop_paradedb_index(index_name, if_exists=True) exists_after = conn.execute( text( @@ -100,7 +100,7 @@ def test_alembic_create_reindex_drop_with_schema(engine): op = Operations(ctx) conn.execute(text("SET LOCAL search_path TO public")) - op.create_bm25_index( + op.create_paradedb_index( index_name, table_name, ["id", "description"], @@ -122,8 +122,8 @@ def test_alembic_create_reindex_drop_with_schema(engine): ).scalar_one() assert exists == 1 - op.reindex_bm25(index_name, schema=schema) - op.drop_bm25_index(index_name, schema=schema, if_exists=True) + op.reindex_paradedb(index_name, schema=schema) + op.drop_paradedb_index(index_name, schema=schema, if_exists=True) exists_after = conn.execute( text( @@ -159,7 +159,9 @@ def _setup_autogen_table(engine, *, with_index: bool = False): conn.execute(text(f'CREATE TABLE "{_AG_TABLE}" (id int primary key, description text not null)')) if with_index: conn.execute( - text(f'CREATE INDEX "{_AG_IDX}" ON "{_AG_TABLE}" USING bm25 (id, description) WITH (key_field=\'id\')') + text( + f'CREATE INDEX "{_AG_IDX}" ON "{_AG_TABLE}" USING paradedb (id, description) WITH (key_field=\'id\')' + ) ) @@ -177,9 +179,9 @@ def _metadata_with_bm25() -> MetaData: Index( _AG_IDX, - BM25Field(t.c.id), - BM25Field(t.c.description), - postgresql_using="bm25", + ParadeDBField(t.c.id), + ParadeDBField(t.c.description), + postgresql_using="paradedb", postgresql_with={"key_field": "id"}, ) return m @@ -193,12 +195,12 @@ def _metadata_without_bm25() -> MetaData: def test_autogenerate_detects_missing_index(engine): - """MetaData has BM25 index but DB does not → CreateBM25IndexOp emitted.""" + """MetaData has BM25 index but DB does not → CreateParadeDBIndexOp emitted.""" _setup_autogen_table(engine, with_index=False) try: upgrade_ops = _run_comparator(engine, _metadata_with_bm25()) - create_ops = [op for op in upgrade_ops.ops if isinstance(op, pdb_alembic.CreateBM25IndexOp)] + create_ops = [op for op in upgrade_ops.ops if isinstance(op, pdb_alembic.CreateParadeDBIndexOp)] assert len(create_ops) == 1 op = create_ops[0] assert op.index_name == _AG_IDX @@ -211,12 +213,12 @@ def test_autogenerate_detects_missing_index(engine): def test_autogenerate_detects_extra_index(engine): - """DB has BM25 index but MetaData does not → DropBM25IndexOp emitted.""" + """DB has BM25 index but MetaData does not → DropParadeDBIndexOp emitted.""" _setup_autogen_table(engine, with_index=True) try: upgrade_ops = _run_comparator(engine, _metadata_without_bm25()) - drop_ops = [op for op in upgrade_ops.ops if isinstance(op, pdb_alembic.DropBM25IndexOp)] + drop_ops = [op for op in upgrade_ops.ops if isinstance(op, pdb_alembic.DropParadeDBIndexOp)] assert any(op.index_name == _AG_IDX for op in drop_ops) finally: _teardown_autogen_table(engine) @@ -231,10 +233,12 @@ def test_autogenerate_no_op_when_indexes_match(engine): # Filter to only ops for our specific test index; the shared engine fixture's # products_bm25_idx may appear as "extra" since our MetaData only knows autogen_test. create_ops = [ - op for op in upgrade_ops.ops if isinstance(op, pdb_alembic.CreateBM25IndexOp) and op.index_name == _AG_IDX + op + for op in upgrade_ops.ops + if isinstance(op, pdb_alembic.CreateParadeDBIndexOp) and op.index_name == _AG_IDX ] drop_ops = [ - op for op in upgrade_ops.ops if isinstance(op, pdb_alembic.DropBM25IndexOp) and op.index_name == _AG_IDX + op for op in upgrade_ops.ops if isinstance(op, pdb_alembic.DropParadeDBIndexOp) and op.index_name == _AG_IDX ] assert not create_ops assert not drop_ops @@ -248,15 +252,15 @@ def test_autogenerate_detects_changed_fields(engine): try: # DB index only covers 'id' with engine.begin() as conn: - conn.execute(text(f'CREATE INDEX "{_AG_IDX}" ON "{_AG_TABLE}" USING bm25 (id) WITH (key_field=\'id\')')) + conn.execute(text(f'CREATE INDEX "{_AG_IDX}" ON "{_AG_TABLE}" USING paradedb (id) WITH (key_field=\'id\')')) # MetaData index covers 'id' and 'description' upgrade_ops = _run_comparator(engine, _metadata_with_bm25()) - drop_ops = [op for op in upgrade_ops.ops if isinstance(op, pdb_alembic.DropBM25IndexOp)] - create_ops = [op for op in upgrade_ops.ops if isinstance(op, pdb_alembic.CreateBM25IndexOp)] - assert any(op.index_name == _AG_IDX for op in drop_ops), "Expected DropBM25IndexOp" - assert any(op.index_name == _AG_IDX for op in create_ops), "Expected CreateBM25IndexOp" + drop_ops = [op for op in upgrade_ops.ops if isinstance(op, pdb_alembic.DropParadeDBIndexOp)] + create_ops = [op for op in upgrade_ops.ops if isinstance(op, pdb_alembic.CreateParadeDBIndexOp)] + assert any(op.index_name == _AG_IDX for op in drop_ops), "Expected DropParadeDBIndexOp" + assert any(op.index_name == _AG_IDX for op in create_ops), "Expected CreateParadeDBIndexOp" finally: _teardown_autogen_table(engine) @@ -272,7 +276,7 @@ def _tokenizer_cast_supported(engine) -> bool: conn.execute( text( f'CREATE INDEX "{index_name}" ON "{table_name}" ' - "USING bm25 (id, (description::pdb.unicode_words('lowercase=true'))) " + "USING paradedb (id, (description::pdb.unicode_words('lowercase=true'))) " "WITH (key_field='id')" ) ) @@ -292,11 +296,11 @@ def _metadata_with_tokenized_bm25() -> MetaData: Index( _AG_IDX, - BM25Field(t.c.id), - BM25Field( + ParadeDBField(t.c.id), + ParadeDBField( t.c.description, tokenizer=tokenizer.simple(options={"alias": "description_simple", "lowercase": True}) ), - postgresql_using="bm25", + postgresql_using="paradedb", postgresql_with={"key_field": "id"}, ) return m @@ -310,14 +314,16 @@ def test_autogenerate_detects_changed_tokenizer_expression(engine): try: with engine.begin() as conn: conn.execute( - text(f'CREATE INDEX "{_AG_IDX}" ON "{_AG_TABLE}" USING bm25 (id, description) WITH (key_field=\'id\')') + text( + f'CREATE INDEX "{_AG_IDX}" ON "{_AG_TABLE}" USING paradedb (id, description) WITH (key_field=\'id\')' + ) ) upgrade_ops = _run_comparator(engine, _metadata_with_tokenized_bm25()) - drop_ops = [op for op in upgrade_ops.ops if isinstance(op, pdb_alembic.DropBM25IndexOp)] - create_ops = [op for op in upgrade_ops.ops if isinstance(op, pdb_alembic.CreateBM25IndexOp)] - assert any(op.index_name == _AG_IDX for op in drop_ops), "Expected DropBM25IndexOp" + drop_ops = [op for op in upgrade_ops.ops if isinstance(op, pdb_alembic.DropParadeDBIndexOp)] + create_ops = [op for op in upgrade_ops.ops if isinstance(op, pdb_alembic.CreateParadeDBIndexOp)] + assert any(op.index_name == _AG_IDX for op in drop_ops), "Expected DropParadeDBIndexOp" create = next(op for op in create_ops if op.index_name == _AG_IDX) assert any("pdb.simple" in expr for expr in create.expressions) assert any("alias=description_simple" in expr for expr in create.expressions) @@ -341,7 +347,7 @@ def _setup_autogen_schema_table(engine, *, with_index: bool = False): conn.execute( text( f'CREATE INDEX "{_AG_SCHEMA_IDX}" ON "{_AG_SCHEMA}"."{_AG_SCHEMA_TABLE}" ' - "USING bm25 (id, description) WITH (key_field='id')" + "USING paradedb (id, description) WITH (key_field='id')" ) ) @@ -359,9 +365,9 @@ def _metadata_with_bm25_in_schema() -> MetaData: Index( _AG_SCHEMA_IDX, - BM25Field(t.c.id), - BM25Field(t.c.description), - postgresql_using="bm25", + ParadeDBField(t.c.id), + ParadeDBField(t.c.description), + postgresql_using="paradedb", postgresql_with={"key_field": "id"}, ) return m @@ -371,7 +377,7 @@ def test_autogenerate_emits_schema_on_create_for_non_default_schema(engine): _setup_autogen_schema_table(engine, with_index=False) try: upgrade_ops = _run_comparator(engine, _metadata_with_bm25_in_schema(), schemas={_AG_SCHEMA}) - create_ops = [op for op in upgrade_ops.ops if isinstance(op, pdb_alembic.CreateBM25IndexOp)] + create_ops = [op for op in upgrade_ops.ops if isinstance(op, pdb_alembic.CreateParadeDBIndexOp)] op = next(o for o in create_ops if o.index_name == _AG_SCHEMA_IDX) assert op.table_schema == _AG_SCHEMA finally: @@ -384,7 +390,7 @@ def test_autogenerate_emits_schema_on_drop_for_non_default_schema(engine): try: metadata = MetaData() upgrade_ops = _run_comparator(engine, metadata, schemas={_AG_SCHEMA}) - drop_ops = [op for op in upgrade_ops.ops if isinstance(op, pdb_alembic.DropBM25IndexOp)] + drop_ops = [op for op in upgrade_ops.ops if isinstance(op, pdb_alembic.DropParadeDBIndexOp)] op = next(o for o in drop_ops if o.index_name == _AG_SCHEMA_IDX) assert op.schema == _AG_SCHEMA finally: @@ -430,7 +436,7 @@ def test_alembic_create_partial_index_with_where_clause(engine): with engine.begin() as conn: ctx = MigrationContext.configure(conn) op = Operations(ctx) - op.create_bm25_index( + op.create_paradedb_index( _PARTIAL_IDX, _PARTIAL_TABLE, ["id", "description", "rating"], @@ -482,15 +488,15 @@ def test_autogenerate_detects_missing_partial_index(engine): Index( _AG_IDX, - BM25Field(t.c.id), - BM25Field(t.c.description), - postgresql_using="bm25", + ParadeDBField(t.c.id), + ParadeDBField(t.c.description), + postgresql_using="paradedb", postgresql_with={"key_field": "id"}, postgresql_where=t.c.id > 2, ) upgrade_ops = _run_comparator(engine, m) - create_ops = [op for op in upgrade_ops.ops if isinstance(op, pdb_alembic.CreateBM25IndexOp)] + create_ops = [op for op in upgrade_ops.ops if isinstance(op, pdb_alembic.CreateParadeDBIndexOp)] assert len(create_ops) == 1 op = create_ops[0] assert op.index_name == _AG_IDX @@ -513,7 +519,7 @@ def test_autogenerate_detects_changed_partial_predicate(engine): conn.execute( text( f'CREATE INDEX "{_AG_IDX}" ON "{_AG_TABLE}" ' - f"USING bm25 (id, description) WITH (key_field='id') WHERE (id > 2)" + f"USING paradedb (id, description) WITH (key_field='id') WHERE (id > 2)" ) ) @@ -524,22 +530,24 @@ def test_autogenerate_detects_changed_partial_predicate(engine): Index( _AG_IDX, - BM25Field(t.c.id), - BM25Field(t.c.description), - postgresql_using="bm25", + ParadeDBField(t.c.id), + ParadeDBField(t.c.description), + postgresql_using="paradedb", postgresql_with={"key_field": "id"}, postgresql_where=t.c.id > 5, ) upgrade_ops = _run_comparator(engine, m) drop_ops = [ - op for op in upgrade_ops.ops if isinstance(op, pdb_alembic.DropBM25IndexOp) and op.index_name == _AG_IDX + op for op in upgrade_ops.ops if isinstance(op, pdb_alembic.DropParadeDBIndexOp) and op.index_name == _AG_IDX ] create_ops = [ - op for op in upgrade_ops.ops if isinstance(op, pdb_alembic.CreateBM25IndexOp) and op.index_name == _AG_IDX + op + for op in upgrade_ops.ops + if isinstance(op, pdb_alembic.CreateParadeDBIndexOp) and op.index_name == _AG_IDX ] - assert len(drop_ops) == 1, "Expected DropBM25IndexOp for predicate change" - assert len(create_ops) == 1, "Expected CreateBM25IndexOp for predicate change" + assert len(drop_ops) == 1, "Expected DropParadeDBIndexOp for predicate change" + assert len(create_ops) == 1, "Expected CreateParadeDBIndexOp for predicate change" assert "5" in create_ops[0].where finally: _teardown_autogen_table(engine) @@ -557,7 +565,7 @@ def test_autogenerate_no_op_when_partial_indexes_match(engine): conn.execute( text( f'CREATE INDEX "{_AG_IDX}" ON "{_AG_TABLE}" ' - f"USING bm25 (id, description) WITH (key_field='id') WHERE (id > 2)" + f"USING paradedb (id, description) WITH (key_field='id') WHERE (id > 2)" ) ) @@ -567,9 +575,9 @@ def test_autogenerate_no_op_when_partial_indexes_match(engine): Index( _AG_IDX, - BM25Field(t.c.id), - BM25Field(t.c.description), - postgresql_using="bm25", + ParadeDBField(t.c.id), + ParadeDBField(t.c.description), + postgresql_using="paradedb", postgresql_with={"key_field": "id"}, postgresql_where=t.c.id > 2, ) @@ -593,7 +601,7 @@ def test_autogenerate_detects_changed_partial_string_literal_case(engine): conn.execute( text( f'CREATE INDEX "{_AG_IDX}" ON "{_AG_TABLE}" ' - f"USING bm25 (id, description) WITH (key_field='id') " + f"USING paradedb (id, description) WITH (key_field='id') " f"WHERE (description = 'ACTIVE')" ) ) @@ -604,22 +612,24 @@ def test_autogenerate_detects_changed_partial_string_literal_case(engine): Index( _AG_IDX, - BM25Field(t.c.id), - BM25Field(t.c.description), - postgresql_using="bm25", + ParadeDBField(t.c.id), + ParadeDBField(t.c.description), + postgresql_using="paradedb", postgresql_with={"key_field": "id"}, postgresql_where="description = 'active'::text", ) upgrade_ops = _run_comparator(engine, m) drop_ops = [ - op for op in upgrade_ops.ops if isinstance(op, pdb_alembic.DropBM25IndexOp) and op.index_name == _AG_IDX + op for op in upgrade_ops.ops if isinstance(op, pdb_alembic.DropParadeDBIndexOp) and op.index_name == _AG_IDX ] create_ops = [ - op for op in upgrade_ops.ops if isinstance(op, pdb_alembic.CreateBM25IndexOp) and op.index_name == _AG_IDX + op + for op in upgrade_ops.ops + if isinstance(op, pdb_alembic.CreateParadeDBIndexOp) and op.index_name == _AG_IDX ] - assert len(drop_ops) == 1, "Expected DropBM25IndexOp for string-literal case change" - assert len(create_ops) == 1, "Expected CreateBM25IndexOp for string-literal case change" + assert len(drop_ops) == 1, "Expected DropParadeDBIndexOp for string-literal case change" + assert len(create_ops) == 1, "Expected CreateParadeDBIndexOp for string-literal case change" finally: _teardown_autogen_table(engine) @@ -633,14 +643,14 @@ def test_autogenerate_round_trip_converges(engine): _setup_autogen_table(engine, with_index=False) try: with engine.begin() as conn: - conn.execute(text(f'CREATE INDEX "{_AG_IDX}" ON "{_AG_TABLE}" USING bm25 (id) WITH (key_field=\'id\')')) + conn.execute(text(f'CREATE INDEX "{_AG_IDX}" ON "{_AG_TABLE}" USING paradedb (id) WITH (key_field=\'id\')')) upgrade_ops = _run_comparator(engine, _metadata_with_bm25()) our_ops = tuple( op for op in upgrade_ops.ops - if (isinstance(op, pdb_alembic.CreateBM25IndexOp) and op.index_name == _AG_IDX) - or (isinstance(op, pdb_alembic.DropBM25IndexOp) and op.index_name == _AG_IDX) + if (isinstance(op, pdb_alembic.CreateParadeDBIndexOp) and op.index_name == _AG_IDX) + or (isinstance(op, pdb_alembic.DropParadeDBIndexOp) and op.index_name == _AG_IDX) ) assert len(our_ops) == 2, f"Expected drop + create ops for drift recovery, got: {our_ops}" @@ -654,8 +664,8 @@ def test_autogenerate_round_trip_converges(engine): remaining_ops = [ op for op in next_upgrade_ops.ops - if (isinstance(op, pdb_alembic.CreateBM25IndexOp) and op.index_name == _AG_IDX) - or (isinstance(op, pdb_alembic.DropBM25IndexOp) and op.index_name == _AG_IDX) + if (isinstance(op, pdb_alembic.CreateParadeDBIndexOp) and op.index_name == _AG_IDX) + or (isinstance(op, pdb_alembic.DropParadeDBIndexOp) and op.index_name == _AG_IDX) ] assert not remaining_ops, f"Expected zero ops after applying drift-recovery ops, got: {remaining_ops}" finally: @@ -687,15 +697,15 @@ def test_alembic_create_reindex_drop_is_queryable(engine): op = Operations(ctx) # Create - op.create_bm25_index(_LIFECYCLE_IDX, _LIFECYCLE_TABLE, ["id", "description"], key_field="id") + op.create_paradedb_index(_LIFECYCLE_IDX, _LIFECYCLE_TABLE, ["id", "description"], key_field="id") _assert_bm25_queryable(conn, _LIFECYCLE_TABLE, _LIFECYCLE_IDX, "description", "running") # Reindex - op.reindex_bm25(_LIFECYCLE_IDX) + op.reindex_paradedb(_LIFECYCLE_IDX) _assert_bm25_queryable(conn, _LIFECYCLE_TABLE, _LIFECYCLE_IDX, "description", "running") # Drop - op.drop_bm25_index(_LIFECYCLE_IDX, if_exists=True) + op.drop_paradedb_index(_LIFECYCLE_IDX, if_exists=True) exists = conn.execute( text("SELECT COUNT(*) FROM pg_indexes WHERE indexname = :idx"), {"idx": _LIFECYCLE_IDX} ).scalar_one() @@ -724,14 +734,14 @@ def test_alembic_reindex_concurrently_autocommit(engine): with engine.begin() as conn: ctx = MigrationContext.configure(conn) op = Operations(ctx) - op.create_bm25_index(_CONC_IDX, _CONC_TABLE, ["id", "description"], key_field="id") + op.create_paradedb_index(_CONC_IDX, _CONC_TABLE, ["id", "description"], key_field="id") # Reindex concurrently requires AUTOCOMMIT autocommit_engine = engine.execution_options(isolation_level="AUTOCOMMIT") with autocommit_engine.connect() as conn: ctx = MigrationContext.configure(conn) op = Operations(ctx) - op.reindex_bm25(_CONC_IDX, concurrently=True) + op.reindex_paradedb(_CONC_IDX, concurrently=True) # Verify index still works with engine.connect() as conn: @@ -753,7 +763,9 @@ def test_autogenerate_detects_changed_key_field(engine): # DB has key_field='id' with engine.begin() as conn: conn.execute( - text(f'CREATE INDEX "{_AG_IDX}" ON "{_AG_TABLE}" USING bm25 (id, description) WITH (key_field=\'id\')') + text( + f'CREATE INDEX "{_AG_IDX}" ON "{_AG_TABLE}" USING paradedb (id, description) WITH (key_field=\'id\')' + ) ) # MetaData declares key_field='description' (different) but keeps the @@ -764,21 +776,23 @@ def test_autogenerate_detects_changed_key_field(engine): Index( _AG_IDX, - BM25Field(t.c.id), - BM25Field(t.c.description), - postgresql_using="bm25", + ParadeDBField(t.c.id), + ParadeDBField(t.c.description), + postgresql_using="paradedb", postgresql_with={"key_field": "description"}, ) upgrade_ops = _run_comparator(engine, m) drop_ops = [ - op for op in upgrade_ops.ops if isinstance(op, pdb_alembic.DropBM25IndexOp) and op.index_name == _AG_IDX + op for op in upgrade_ops.ops if isinstance(op, pdb_alembic.DropParadeDBIndexOp) and op.index_name == _AG_IDX ] create_ops = [ - op for op in upgrade_ops.ops if isinstance(op, pdb_alembic.CreateBM25IndexOp) and op.index_name == _AG_IDX + op + for op in upgrade_ops.ops + if isinstance(op, pdb_alembic.CreateParadeDBIndexOp) and op.index_name == _AG_IDX ] - assert len(drop_ops) == 1, "Expected DropBM25IndexOp for key_field change" - assert len(create_ops) == 1, "Expected CreateBM25IndexOp for key_field change" + assert len(drop_ops) == 1, "Expected DropParadeDBIndexOp for key_field change" + assert len(create_ops) == 1, "Expected CreateParadeDBIndexOp for key_field change" assert create_ops[0].key_field == "description" finally: _teardown_autogen_table(engine) @@ -804,7 +818,7 @@ def test_alembic_expression_index_lifecycle(engine): with engine.begin() as conn: ctx = MigrationContext.configure(conn) op = Operations(ctx) - op.create_bm25_index( + op.create_paradedb_index( _EXPR_IDX, _EXPR_TABLE, ["id", "((description)::pdb.simple('alias=desc_simple,lowercase=true'))"], @@ -818,7 +832,7 @@ def test_alembic_expression_index_lifecycle(engine): assert "pdb.simple" in indexdef assert "desc_simple" in indexdef - op.drop_bm25_index(_EXPR_IDX, if_exists=True) + op.drop_paradedb_index(_EXPR_IDX, if_exists=True) exists = conn.execute( text("SELECT COUNT(*) FROM pg_indexes WHERE indexname = :idx"), {"idx": _EXPR_IDX} ).scalar_one() @@ -850,7 +864,7 @@ def test_alembic_multi_tokenizer_expression_lifecycle(engine): with engine.begin() as conn: ctx = MigrationContext.configure(conn) op = Operations(ctx) - op.create_bm25_index( + op.create_paradedb_index( _MULTI_IDX, _MULTI_TABLE, [ diff --git a/tests/integration/test_indexing_integration.py b/tests/integration/test_indexing_integration.py index bd55da4..09d8266 100644 --- a/tests/integration/test_indexing_integration.py +++ b/tests/integration/test_indexing_integration.py @@ -7,7 +7,7 @@ from paradedb.sqlalchemy import pdb from paradedb.sqlalchemy.expr import json_text -from paradedb.sqlalchemy.indexing import BM25Field, assert_indexed, describe +from paradedb.sqlalchemy.indexing import ParadeDBField, assert_indexed, describe from paradedb.sqlalchemy import tokenizer from paradedb.sqlalchemy.errors import FieldNotIndexedError @@ -34,7 +34,7 @@ def _tokenizer_cast_supported(engine) -> bool: f""" CREATE INDEX {index_name} ON {table_name} - USING bm25 (id, (description::pdb.unicode_words('lowercase=true'))) + USING paradedb (id, (description::pdb.unicode_words('lowercase=true'))) WITH (key_field='id') """ ) @@ -63,10 +63,10 @@ def test_bm25_index_create(engine): idx = Index( index_name, - BM25Field(products.c.id), - BM25Field(products.c.description), - BM25Field(products.c.category), - postgresql_using="bm25", + ParadeDBField(products.c.id), + ParadeDBField(products.c.description), + ParadeDBField(products.c.category), + postgresql_using="paradedb", postgresql_with={"key_field": "id"}, ) idx.create(engine) @@ -84,7 +84,7 @@ def test_bm25_index_create(engine): {"table_name": table_name, "index_name": index_name}, ).one() - assert "USING bm25" in row.indexdef + assert "USING paradedb" in row.indexdef assert "description" in row.indexdef assert "category" in row.indexdef @@ -111,10 +111,10 @@ def test_bm25_index_with_tokenizers_when_supported(engine): idx = Index( index_name, - BM25Field(products.c.id), - BM25Field(products.c.description, tokenizer=tokenizer.unicode_words(options={"lowercase": True})), - BM25Field(products.c.category, tokenizer=tokenizer.literal_normalized(options={"alias": "category_exact"})), - postgresql_using="bm25", + ParadeDBField(products.c.id), + ParadeDBField(products.c.description, tokenizer=tokenizer.unicode_words(options={"lowercase": True})), + ParadeDBField(products.c.category, tokenizer=tokenizer.literal_normalized(options={"alias": "category_exact"})), + postgresql_using="paradedb", postgresql_with={"key_field": "id"}, ) idx.create(engine) @@ -158,16 +158,16 @@ def test_bm25_index_json_keys_when_supported(engine): idx = Index( index_name, - BM25Field(products.c.id), - BM25Field( + ParadeDBField(products.c.id), + ParadeDBField( json_text(products.c.metadata, "color"), tokenizer=tokenizer.literal(options={"alias": "metadata_color"}), ), - BM25Field( + ParadeDBField( json_text(products.c.metadata, "location"), tokenizer=tokenizer.literal(options={"alias": "metadata_location"}), ), - postgresql_using="bm25", + postgresql_using="paradedb", postgresql_with={"key_field": "id"}, ) idx.create(engine) @@ -210,10 +210,10 @@ def test_bm25_index_non_text_expression_with_pdb_alias(engine): idx = Index( index_name, - BM25Field(products.c.id), - BM25Field(products.c.description), - BM25Field(pdb.alias(products.c.rating + 1, "rating_plus_one")), - postgresql_using="bm25", + ParadeDBField(products.c.id), + ParadeDBField(products.c.description), + ParadeDBField(pdb.alias(products.c.rating + 1, "rating_plus_one")), + postgresql_using="paradedb", postgresql_with={"key_field": "id"}, ) idx.create(engine) @@ -251,9 +251,9 @@ def test_create_all_with_attached_bm25_index(engine): ) Index( index_name, - BM25Field(items.c.id), - BM25Field(items.c.description), - postgresql_using="bm25", + ParadeDBField(items.c.id), + ParadeDBField(items.c.description), + postgresql_using="paradedb", postgresql_with={"key_field": "id"}, ) metadata.create_all(engine) @@ -292,13 +292,13 @@ def test_duplicate_tokenizer_alias_is_rejected(engine): idx = Index( index_name, - BM25Field(products.c.id), - BM25Field( + ParadeDBField(products.c.id), + ParadeDBField( products.c.description, tokenizer=tokenizer.unicode_words(options={"alias": "desc_alias", "lowercase": True}), ), - BM25Field(products.c.description, tokenizer=tokenizer.literal(options={"alias": "desc_alias"})), - postgresql_using="bm25", + ParadeDBField(products.c.description, tokenizer=tokenizer.literal(options={"alias": "desc_alias"})), + postgresql_using="paradedb", postgresql_with={"key_field": "id"}, ) @@ -324,9 +324,9 @@ def test_missing_key_field_is_rejected(engine): idx = Index( index_name, - BM25Field(products.c.id), - BM25Field(products.c.description), - postgresql_using="bm25", + ParadeDBField(products.c.id), + ParadeDBField(products.c.description), + postgresql_using="paradedb", ) with pytest.raises(ValueError, match="key_field"): @@ -352,10 +352,10 @@ def test_describe_returns_fields_and_aliases(engine): idx = Index( index_name, - BM25Field(products.c.id), - BM25Field(products.c.description, tokenizer=tokenizer.unicode_words(options={"lowercase": True})), - BM25Field(products.c.category, tokenizer=tokenizer.literal_normalized(options={"alias": "category_exact"})), - postgresql_using="bm25", + ParadeDBField(products.c.id), + ParadeDBField(products.c.description, tokenizer=tokenizer.unicode_words(options={"lowercase": True})), + ParadeDBField(products.c.category, tokenizer=tokenizer.literal_normalized(options={"alias": "category_exact"})), + postgresql_using="paradedb", postgresql_with={"key_field": "id"}, ) idx.create(engine) @@ -391,10 +391,10 @@ def test_describe_includes_tokenizers(engine): idx = Index( index_name, - BM25Field(products.c.id), - BM25Field(products.c.description, tokenizer=tokenizer.unicode_words(options={"lowercase": True})), - BM25Field(products.c.category, tokenizer=tokenizer.literal()), - postgresql_using="bm25", + ParadeDBField(products.c.id), + ParadeDBField(products.c.description, tokenizer=tokenizer.unicode_words(options={"lowercase": True})), + ParadeDBField(products.c.category, tokenizer=tokenizer.literal()), + postgresql_using="paradedb", postgresql_with={"key_field": "id"}, ) idx.create(engine) @@ -430,12 +430,12 @@ def test_describe_and_assert_indexed_for_json_expression_tokenizer(engine): idx = Index( index_name, - BM25Field(products.c.id), - BM25Field( + ParadeDBField(products.c.id), + ParadeDBField( json_text(products.c.metadata, "color"), tokenizer=tokenizer.literal(options={"alias": "metadata_color"}), ), - postgresql_using="bm25", + postgresql_using="paradedb", postgresql_with={"key_field": "id"}, ) idx.create(engine) @@ -472,9 +472,9 @@ def test_assert_indexed_passes_and_raises(engine): idx = Index( index_name, - BM25Field(tbl.c.id), - BM25Field(tbl.c.description), - postgresql_using="bm25", + ParadeDBField(tbl.c.id), + ParadeDBField(tbl.c.description), + postgresql_using="paradedb", postgresql_with={"key_field": "id"}, ) idx.create(engine) @@ -511,9 +511,9 @@ def test_describe_and_assert_indexed_with_explicit_schema(engine): idx = Index( index_name, - BM25Field(products.c.id), - BM25Field(products.c.description), - postgresql_using="bm25", + ParadeDBField(products.c.id), + ParadeDBField(products.c.description), + postgresql_using="paradedb", postgresql_with={"key_field": "id"}, ) idx.create(engine) @@ -559,9 +559,9 @@ def test_bm25_partial_index_generates_where_clause(engine): idx = Index( index_name, - BM25Field(products.c.id), - BM25Field(products.c.description), - postgresql_using="bm25", + ParadeDBField(products.c.id), + ParadeDBField(products.c.description), + postgresql_using="paradedb", postgresql_with={"key_field": "id"}, postgresql_where=products.c.rating > 3, ) @@ -605,9 +605,9 @@ def test_bm25_partial_index_filters_search_results(engine): # Only index rows where rating > 3 idx = Index( index_name, - BM25Field(products.c.id), - BM25Field(products.c.description), - postgresql_using="bm25", + ParadeDBField(products.c.id), + ParadeDBField(products.c.description), + postgresql_using="paradedb", postgresql_with={"key_field": "id"}, postgresql_where=products.c.rating > 3, ) @@ -655,9 +655,9 @@ def test_bm25_index_create_concurrently(engine): idx = Index( index_name, - BM25Field(products.c.id), - BM25Field(products.c.description), - postgresql_using="bm25", + ParadeDBField(products.c.id), + ParadeDBField(products.c.description), + postgresql_using="paradedb", postgresql_with={"key_field": "id"}, postgresql_concurrently=True, ) diff --git a/tests/integration/test_paradedb_queries_integration.py b/tests/integration/test_paradedb_queries_integration.py index d13ab57..1322a6b 100644 --- a/tests/integration/test_paradedb_queries_integration.py +++ b/tests/integration/test_paradedb_queries_integration.py @@ -504,7 +504,9 @@ def test_range_term_with_range_type(mock_session): with engine.begin() as conn: conn.execute( - text("CREATE INDEX rt_items_pq_bm25_idx ON rt_items_pq USING bm25 (id, weight_range) WITH (key_field='id')") + text( + "CREATE INDEX rt_items_pq_bm25_idx ON rt_items_pq USING paradedb (id, weight_range) WITH (key_field='id')" + ) ) conn.execute( text( @@ -555,7 +557,7 @@ def test_range_term_scalar_contains_point(mock_session): with engine.begin() as conn: conn.execute( text( - "CREATE INDEX rt_scalar_items_pq_bm25_idx ON rt_scalar_items_pq USING bm25 (id, weight_range) " + "CREATE INDEX rt_scalar_items_pq_bm25_idx ON rt_scalar_items_pq USING paradedb (id, weight_range) " "WITH (key_field='id')" ) ) @@ -652,7 +654,7 @@ def test_exists_query_matches_non_null(mock_session): with engine.begin() as conn: conn.execute( text( - "CREATE INDEX exists_items_pq_bm25_idx ON exists_items_pq USING bm25 (id, rating) WITH (key_field='id')" + "CREATE INDEX exists_items_pq_bm25_idx ON exists_items_pq USING paradedb (id, rating) WITH (key_field='id')" ) ) conn.execute(text("INSERT INTO exists_items_pq (id, rating) VALUES (1, 5), (2, NULL), (3, 0)")) diff --git a/tests/integration/test_phase0_modules_integration.py b/tests/integration/test_phase0_modules_integration.py index 3bb15a8..038df53 100644 --- a/tests/integration/test_phase0_modules_integration.py +++ b/tests/integration/test_phase0_modules_integration.py @@ -10,7 +10,7 @@ from paradedb.sqlalchemy import inspect as pdb_inspect from paradedb.sqlalchemy import search from paradedb.sqlalchemy.errors import DuplicateTokenizerAliasError, InvalidArgumentError -from paradedb.sqlalchemy.indexing import BM25Field +from paradedb.sqlalchemy.indexing import ParadeDBField from paradedb.sqlalchemy import tokenizer import paradedb.sqlalchemy.alembic # noqa: F401 Ensures Alembic ops registration @@ -59,10 +59,10 @@ def test_custom_errors_raised_for_validation(engine): idx = Index( index_name, - BM25Field(products.c.id), - BM25Field(products.c.description, tokenizer=tokenizer.unicode_words(options={"alias": "dup"})), - BM25Field(products.c.description, tokenizer=tokenizer.literal(options={"alias": "dup"})), - postgresql_using="bm25", + ParadeDBField(products.c.id), + ParadeDBField(products.c.description, tokenizer=tokenizer.unicode_words(options={"alias": "dup"})), + ParadeDBField(products.c.description, tokenizer=tokenizer.literal(options={"alias": "dup"})), + postgresql_using="paradedb", postgresql_with={"key_field": "id"}, ) @@ -90,7 +90,7 @@ def test_alembic_ops_create_reindex_drop(engine): ctx = MigrationContext.configure(conn) op = Operations(ctx) - op.create_bm25_index(index_name, table_name, ["id", "description"], key_field="id") + op.create_paradedb_index(index_name, table_name, ["id", "description"], key_field="id") exists = conn.execute( text( @@ -105,8 +105,8 @@ def test_alembic_ops_create_reindex_drop(engine): ).scalar_one() assert exists == 1 - op.reindex_bm25(index_name) - op.drop_bm25_index(index_name, if_exists=True) + op.reindex_paradedb(index_name) + op.drop_paradedb_index(index_name, if_exists=True) exists_after_drop = conn.execute( text( diff --git a/tests/integration/test_range_query_integration.py b/tests/integration/test_range_query_integration.py index c0262b1..40e8d65 100644 --- a/tests/integration/test_range_query_integration.py +++ b/tests/integration/test_range_query_integration.py @@ -30,7 +30,7 @@ def test_range_query_with_op_and_all_predicate(engine): with engine.begin() as conn: conn.execute( text( - "CREATE INDEX range_items_bm25_idx ON range_items USING bm25 (id, description, weight_range) WITH (key_field='id')" + "CREATE INDEX range_items_bm25_idx ON range_items USING paradedb (id, description, weight_range) WITH (key_field='id')" ) ) conn.execute( diff --git a/tests/unit/test_alembic_unit.py b/tests/unit/test_alembic_unit.py index 399b3be..c16c9cb 100644 --- a/tests/unit/test_alembic_unit.py +++ b/tests/unit/test_alembic_unit.py @@ -8,7 +8,7 @@ from sqlalchemy import Column, Integer, MetaData, Table, Text import paradedb.sqlalchemy.alembic as pdb_alembic -from paradedb.sqlalchemy.indexing import BM25Field +from paradedb.sqlalchemy.indexing import ParadeDBField class DummyOps: @@ -22,70 +22,70 @@ def execute(self, sql: str) -> None: def test_create_drop_reindex_sql_generation(): ops = DummyOps() - create_op = pdb_alembic.CreateBM25IndexOp( + create_op = pdb_alembic.CreateParadeDBIndexOp( index_name='idx "quoted"', table_name='tbl "quoted"', expressions=["id", "description"], key_field="id", ) - pdb_alembic._create_bm25_index_impl(ops, create_op) + pdb_alembic._create_paradedb_index_impl(ops, create_op) assert ( ops.sql[-1] - == 'CREATE INDEX "idx ""quoted""" ON "tbl ""quoted""" USING bm25 (id, description) WITH (key_field=\'id\')' + == 'CREATE INDEX "idx ""quoted""" ON "tbl ""quoted""" USING paradedb (id, description) WITH (key_field=\'id\')' ) - drop_op = pdb_alembic.DropBM25IndexOp(index_name='idx "quoted"', if_exists=True) - pdb_alembic._drop_bm25_index_impl(ops, drop_op) + drop_op = pdb_alembic.DropParadeDBIndexOp(index_name='idx "quoted"', if_exists=True) + pdb_alembic._drop_paradedb_index_impl(ops, drop_op) assert ops.sql[-1] == 'DROP INDEX IF EXISTS "idx ""quoted"""' - reindex_op = pdb_alembic.ReindexBM25Op(index_name='idx "quoted"', concurrently=True) - pdb_alembic._reindex_bm25_impl(ops, reindex_op) + reindex_op = pdb_alembic.ReindexParadeDBOp(index_name='idx "quoted"', concurrently=True) + pdb_alembic._reindex_paradedb_impl(ops, reindex_op) assert ops.sql[-1] == 'REINDEX INDEX CONCURRENTLY "idx ""quoted"""' def test_create_sql_generation_preserves_tokenizer_expression(): ops = DummyOps() - create_op = pdb_alembic.CreateBM25IndexOp( + create_op = pdb_alembic.CreateParadeDBIndexOp( index_name="products_bm25_idx", table_name="products", expressions=["id", "((description)::pdb.simple('alias=description_simple,lowercase=true'))"], key_field="id", ) - pdb_alembic._create_bm25_index_impl(ops, create_op) + pdb_alembic._create_paradedb_index_impl(ops, create_op) assert ops.sql[-1] == ( 'CREATE INDEX "products_bm25_idx" ON "products" ' - "USING bm25 (id, ((description)::pdb.simple('alias=description_simple,lowercase=true'))) " + "USING paradedb (id, ((description)::pdb.simple('alias=description_simple,lowercase=true'))) " "WITH (key_field='id')" ) def test_create_drop_reindex_sql_generation_with_schema(): ops = DummyOps() - create_op = pdb_alembic.CreateBM25IndexOp( + create_op = pdb_alembic.CreateParadeDBIndexOp( index_name="products_bm25_idx", table_name="products", expressions=["id", "description"], key_field="id", table_schema="analytics", ) - pdb_alembic._create_bm25_index_impl(ops, create_op) + pdb_alembic._create_paradedb_index_impl(ops, create_op) assert ops.sql[-1] == ( 'CREATE INDEX "products_bm25_idx" ON "analytics"."products" ' - "USING bm25 (id, description) WITH (key_field='id')" + "USING paradedb (id, description) WITH (key_field='id')" ) - drop_op = pdb_alembic.DropBM25IndexOp(index_name="products_bm25_idx", if_exists=True, schema="analytics") - pdb_alembic._drop_bm25_index_impl(ops, drop_op) + drop_op = pdb_alembic.DropParadeDBIndexOp(index_name="products_bm25_idx", if_exists=True, schema="analytics") + pdb_alembic._drop_paradedb_index_impl(ops, drop_op) assert ops.sql[-1] == 'DROP INDEX IF EXISTS "analytics"."products_bm25_idx"' - reindex_op = pdb_alembic.ReindexBM25Op(index_name="products_bm25_idx", concurrently=True, schema="analytics") - pdb_alembic._reindex_bm25_impl(ops, reindex_op) + reindex_op = pdb_alembic.ReindexParadeDBOp(index_name="products_bm25_idx", concurrently=True, schema="analytics") + pdb_alembic._reindex_paradedb_impl(ops, reindex_op) assert ops.sql[-1] == 'REINDEX INDEX CONCURRENTLY "analytics"."products_bm25_idx"' -def test_create_bm25_index_rejects_removed_index_schema_kwarg(): +def test_create_paradedb_index_rejects_removed_index_schema_kwarg(): with pytest.raises(TypeError, match="index_schema"): - pdb_alembic.CreateBM25IndexOp.create_bm25_index( + pdb_alembic.CreateParadeDBIndexOp.create_paradedb_index( object(), "products_bm25_idx", "products", @@ -95,8 +95,8 @@ def test_create_bm25_index_rejects_removed_index_schema_kwarg(): ) -def test_create_bm25_index_reverse_returns_drop_op(): - create_op = pdb_alembic.CreateBM25IndexOp( +def test_create_paradedb_index_reverse_returns_drop_op(): + create_op = pdb_alembic.CreateParadeDBIndexOp( index_name="products_bm25_idx", table_name="products", expressions=["id", "description"], @@ -106,14 +106,14 @@ def test_create_bm25_index_reverse_returns_drop_op(): reversed_op = create_op.reverse() - assert isinstance(reversed_op, pdb_alembic.DropBM25IndexOp) + assert isinstance(reversed_op, pdb_alembic.DropParadeDBIndexOp) assert reversed_op.index_name == "products_bm25_idx" assert reversed_op.schema == "analytics" assert reversed_op.if_exists is True -def test_drop_bm25_index_reverse_returns_create_op_when_metadata_present(): - drop_op = pdb_alembic.DropBM25IndexOp( +def test_drop_paradedb_index_reverse_returns_create_op_when_metadata_present(): + drop_op = pdb_alembic.DropParadeDBIndexOp( index_name="products_bm25_idx", if_exists=True, schema="analytics", @@ -125,7 +125,7 @@ def test_drop_bm25_index_reverse_returns_create_op_when_metadata_present(): reversed_op = drop_op.reverse() - assert isinstance(reversed_op, pdb_alembic.CreateBM25IndexOp) + assert isinstance(reversed_op, pdb_alembic.CreateParadeDBIndexOp) assert reversed_op.index_name == "products_bm25_idx" assert reversed_op.table_name == "products" assert reversed_op.expressions == ["id", "description"] @@ -134,8 +134,8 @@ def test_drop_bm25_index_reverse_returns_create_op_when_metadata_present(): assert reversed_op.where == "rating > 3" -def test_drop_bm25_index_reverse_raises_without_recreate_metadata(): - drop_op = pdb_alembic.DropBM25IndexOp(index_name="products_bm25_idx", if_exists=True, schema="analytics") +def test_drop_paradedb_index_reverse_raises_without_recreate_metadata(): + drop_op = pdb_alembic.DropParadeDBIndexOp(index_name="products_bm25_idx", if_exists=True, schema="analytics") with pytest.raises(NotImplementedError, match="requires recreate metadata"): drop_op.reverse() @@ -144,7 +144,7 @@ def test_drop_bm25_index_reverse_raises_without_recreate_metadata(): def test_upgrade_ops_reverse_into_handles_bm25_create_op(): upgrade_ops = UpgradeOps( [ - pdb_alembic.CreateBM25IndexOp( + pdb_alembic.CreateParadeDBIndexOp( index_name="products_bm25_idx", table_name="products", expressions=["id", "description"], @@ -158,7 +158,7 @@ def test_upgrade_ops_reverse_into_handles_bm25_create_op(): assert len(downgrade_ops.ops) == 1 reversed_op = downgrade_ops.ops[0] - assert isinstance(reversed_op, pdb_alembic.DropBM25IndexOp) + assert isinstance(reversed_op, pdb_alembic.DropParadeDBIndexOp) assert reversed_op.index_name == "products_bm25_idx" assert reversed_op.schema == "analytics" assert reversed_op.if_exists is True @@ -167,7 +167,7 @@ def test_upgrade_ops_reverse_into_handles_bm25_create_op(): def test_upgrade_ops_reverse_into_handles_bm25_drop_op_with_recreate_metadata(): upgrade_ops = UpgradeOps( [ - pdb_alembic.DropBM25IndexOp( + pdb_alembic.DropParadeDBIndexOp( index_name="products_bm25_idx", if_exists=True, schema="analytics", @@ -183,7 +183,7 @@ def test_upgrade_ops_reverse_into_handles_bm25_drop_op_with_recreate_metadata(): assert len(downgrade_ops.ops) == 1 reversed_op = downgrade_ops.ops[0] - assert isinstance(reversed_op, pdb_alembic.CreateBM25IndexOp) + assert isinstance(reversed_op, pdb_alembic.CreateParadeDBIndexOp) assert reversed_op.index_name == "products_bm25_idx" assert reversed_op.table_name == "products" assert reversed_op.expressions == ["id", "description"] @@ -198,7 +198,7 @@ def test_alembic_renderers_registered_and_emit_python(): create_lines = render_op( autogen_ctx, - pdb_alembic.CreateBM25IndexOp( + pdb_alembic.CreateParadeDBIndexOp( index_name="products_bm25_idx", table_name="products", expressions=["id", "description"], @@ -206,18 +206,18 @@ def test_alembic_renderers_registered_and_emit_python(): ), ) assert create_lines == [ - "op.create_bm25_index('products_bm25_idx', 'products', ['id', 'description'], key_field='id')" + "op.create_paradedb_index('products_bm25_idx', 'products', ['id', 'description'], key_field='id')" ] drop_lines = render_op( autogen_ctx, - pdb_alembic.DropBM25IndexOp(index_name="products_bm25_idx", if_exists=False), + pdb_alembic.DropParadeDBIndexOp(index_name="products_bm25_idx", if_exists=False), ) - assert drop_lines == ["op.drop_bm25_index('products_bm25_idx', if_exists=False)"] + assert drop_lines == ["op.drop_paradedb_index('products_bm25_idx', if_exists=False)"] drop_lines_with_recreate = render_op( autogen_ctx, - pdb_alembic.DropBM25IndexOp( + pdb_alembic.DropParadeDBIndexOp( index_name="products_bm25_idx", if_exists=False, schema="analytics", @@ -228,18 +228,18 @@ def test_alembic_renderers_registered_and_emit_python(): ), ) assert drop_lines_with_recreate == [ - "op.drop_bm25_index('products_bm25_idx', if_exists=False, schema='analytics', table_name='products', expressions=['id', 'description'], key_field='id', where='rating > 3')" + "op.drop_paradedb_index('products_bm25_idx', if_exists=False, schema='analytics', table_name='products', expressions=['id', 'description'], key_field='id', where='rating > 3')" ] reindex_lines = render_op( autogen_ctx, - pdb_alembic.ReindexBM25Op(index_name="products_bm25_idx", concurrently=True), + pdb_alembic.ReindexParadeDBOp(index_name="products_bm25_idx", concurrently=True), ) - assert reindex_lines == ["op.reindex_bm25('products_bm25_idx', concurrently=True)"] + assert reindex_lines == ["op.reindex_paradedb('products_bm25_idx', concurrently=True)"] create_lines_with_schema = render_op( autogen_ctx, - pdb_alembic.CreateBM25IndexOp( + pdb_alembic.CreateParadeDBIndexOp( index_name="products_bm25_idx", table_name="products", expressions=["id", "description"], @@ -248,7 +248,7 @@ def test_alembic_renderers_registered_and_emit_python(): ), ) assert create_lines_with_schema == [ - "op.create_bm25_index('products_bm25_idx', 'products', ['id', 'description'], key_field='id', table_schema='analytics')" + "op.create_paradedb_index('products_bm25_idx', 'products', ['id', 'description'], key_field='id', table_schema='analytics')" ] @@ -265,9 +265,9 @@ def _make_metadata_with_bm25() -> tuple[MetaData, object]: t = Table("products", m, Column("id", Integer), Column("description", Text)) bm25_idx = Index( "products_bm25_idx", - BM25Field(t.c.id), - BM25Field(t.c.description), - postgresql_using="bm25", + ParadeDBField(t.c.id), + ParadeDBField(t.c.description), + postgresql_using="paradedb", postgresql_with={"key_field": "id"}, ) # A regular (non-BM25) index on the same table @@ -277,7 +277,7 @@ def _make_metadata_with_bm25() -> tuple[MetaData, object]: def test_autogen_meta_indexes_finds_bm25_only(): m, bm25_idx = _make_metadata_with_bm25() - result = pdb_alembic._autogen_bm25_meta_indexes(m, {"public"}, default_schema="public") + result = pdb_alembic._autogen_paradedb_meta_indexes(m, {"public"}, default_schema="public") assert ("public", "products_bm25_idx") in result # Regular index must not appear @@ -292,18 +292,18 @@ def test_autogen_meta_indexes_schema_filter(): t = Table("things", m, Column("id", Integer), Column("body", Text), schema="other") Index( "things_bm25_idx", - BM25Field(t.c.id), - BM25Field(t.c.body), - postgresql_using="bm25", + ParadeDBField(t.c.id), + ParadeDBField(t.c.body), + postgresql_using="paradedb", postgresql_with={"key_field": "id"}, ) # Only looking at schema "public" — the "other" table's index must not appear - result = pdb_alembic._autogen_bm25_meta_indexes(m, {"public"}, default_schema="public") + result = pdb_alembic._autogen_paradedb_meta_indexes(m, {"public"}, default_schema="public") assert ("other", "things_bm25_idx") not in result # When we look at "other", it should appear - result2 = pdb_alembic._autogen_bm25_meta_indexes(m, {"other"}, default_schema="public") + result2 = pdb_alembic._autogen_paradedb_meta_indexes(m, {"other"}, default_schema="public") assert ("other", "things_bm25_idx") in result2 @@ -314,20 +314,20 @@ def test_autogen_meta_indexes_uses_explicit_default_schema_for_unschematized_tab t = Table("products", m, Column("id", Integer), Column("description", Text)) Index( "products_bm25_idx", - BM25Field(t.c.id), - BM25Field(t.c.description), - postgresql_using="bm25", + ParadeDBField(t.c.id), + ParadeDBField(t.c.description), + postgresql_using="paradedb", postgresql_with={"key_field": "id"}, ) - result_public = pdb_alembic._autogen_bm25_meta_indexes(m, {"public", "other"}, default_schema="public") + result_public = pdb_alembic._autogen_paradedb_meta_indexes(m, {"public", "other"}, default_schema="public") assert ("public", "products_bm25_idx") in result_public - result_other = pdb_alembic._autogen_bm25_meta_indexes(m, {"public", "other"}, default_schema="other") + result_other = pdb_alembic._autogen_paradedb_meta_indexes(m, {"public", "other"}, default_schema="other") assert ("other", "products_bm25_idx") in result_other -def test_suppress_standard_bm25_ops_removes_from_modify_table_ops(): +def test_suppress_standard_paradedb_ops_removes_from_modify_table_ops(): """Ops for BM25 indexes inside ModifyTableOps are removed; non-BM25 ops survive.""" m = MetaData() t = Table("products", m, Column("id", Integer), Column("description", Text)) @@ -339,7 +339,7 @@ def test_suppress_standard_bm25_ops_removes_from_modify_table_ops(): modify_ops = ModifyTableOps("products", [bm25_idx, regular_idx, drop_bm25], schema=None) upgrade_ops = UpgradeOps([modify_ops]) - pdb_alembic._suppress_standard_bm25_ops(upgrade_ops, {"products_bm25_idx"}) + pdb_alembic._suppress_standard_paradedb_ops(upgrade_ops, {"products_bm25_idx"}) # ModifyTableOps container is still there assert len(upgrade_ops.ops) == 1 @@ -349,7 +349,7 @@ def test_suppress_standard_bm25_ops_removes_from_modify_table_ops(): assert remaining[0].index_name == "products_desc_idx" -def test_suppress_standard_bm25_ops_removes_top_level(): +def test_suppress_standard_paradedb_ops_removes_top_level(): """Top-level CreateIndexOp/DropIndexOp for BM25 indexes are also removed.""" m = MetaData() t = Table("products", m, Column("id", Integer)) @@ -358,13 +358,13 @@ def test_suppress_standard_bm25_ops_removes_top_level(): regular_create = CreateIndexOp("regular_idx", "products", [t.c.id]) upgrade_ops = UpgradeOps([bm25_create, regular_create]) - pdb_alembic._suppress_standard_bm25_ops(upgrade_ops, {"bm25_idx"}) + pdb_alembic._suppress_standard_paradedb_ops(upgrade_ops, {"bm25_idx"}) assert len(upgrade_ops.ops) == 1 assert upgrade_ops.ops[0].index_name == "regular_idx" -def test_suppress_standard_bm25_ops_noop_when_no_bm25(): +def test_suppress_standard_paradedb_ops_noop_when_no_bm25(): """When there are no BM25 indexes to suppress, ops are unchanged.""" m = MetaData() t = Table("products", m, Column("id", Integer)) @@ -372,19 +372,19 @@ def test_suppress_standard_bm25_ops_noop_when_no_bm25(): regular_create = CreateIndexOp("regular_idx", "products", [t.c.id]) upgrade_ops = UpgradeOps([regular_create]) - pdb_alembic._suppress_standard_bm25_ops(upgrade_ops, set()) + pdb_alembic._suppress_standard_paradedb_ops(upgrade_ops, set()) assert len(upgrade_ops.ops) == 1 -def test_normalize_bm25_expression_keeps_dotted_literal_content(): +def test_normalize_paradedb_expression_keeps_dotted_literal_content(): expr = "(description)::pdb.regex_pattern('run.*')" - normalized = pdb_alembic._normalize_bm25_expression(expr) + normalized = pdb_alembic._normalize_paradedb_expression(expr) assert normalized == "(description)::pdb.regex_pattern('run.*')" -def test_normalize_bm25_expression_strips_relation_qualifiers_only(): +def test_normalize_paradedb_expression_strips_relation_qualifiers_only(): expr = '"public"."products"."description"::pdb.simple(\'alias=description_simple\')' - normalized = pdb_alembic._normalize_bm25_expression(expr) + normalized = pdb_alembic._normalize_paradedb_expression(expr) assert normalized == "description::pdb.simple('alias=description_simple')" @@ -410,33 +410,33 @@ def test_strip_relation_qualifiers_avoids_substring_false_positive(): def test_create_sql_generation_with_where_clause(): ops = DummyOps() - create_op = pdb_alembic.CreateBM25IndexOp( + create_op = pdb_alembic.CreateParadeDBIndexOp( index_name="products_bm25_idx", table_name="products", expressions=["id", "description"], key_field="id", where="rating > 3", ) - pdb_alembic._create_bm25_index_impl(ops, create_op) + pdb_alembic._create_paradedb_index_impl(ops, create_op) assert ops.sql[-1] == ( 'CREATE INDEX "products_bm25_idx" ON "products" ' - "USING bm25 (id, description) WITH (key_field='id') WHERE rating > 3" + "USING paradedb (id, description) WITH (key_field='id') WHERE rating > 3" ) def test_create_sql_generation_without_where_clause(): """When where is None, no WHERE suffix is appended.""" ops = DummyOps() - create_op = pdb_alembic.CreateBM25IndexOp( + create_op = pdb_alembic.CreateParadeDBIndexOp( index_name="products_bm25_idx", table_name="products", expressions=["id", "description"], key_field="id", ) - pdb_alembic._create_bm25_index_impl(ops, create_op) + pdb_alembic._create_paradedb_index_impl(ops, create_op) assert ( ops.sql[-1] - == 'CREATE INDEX "products_bm25_idx" ON "products" USING bm25 (id, description) WITH (key_field=\'id\')' + == 'CREATE INDEX "products_bm25_idx" ON "products" USING paradedb (id, description) WITH (key_field=\'id\')' ) @@ -446,7 +446,7 @@ def test_renderer_emits_where_kwarg(): lines = render_op( autogen_ctx, - pdb_alembic.CreateBM25IndexOp( + pdb_alembic.CreateParadeDBIndexOp( index_name="products_bm25_idx", table_name="products", expressions=["id", "description"], @@ -464,7 +464,7 @@ def test_renderer_omits_where_when_none(): lines = render_op( autogen_ctx, - pdb_alembic.CreateBM25IndexOp( + pdb_alembic.CreateParadeDBIndexOp( index_name="products_bm25_idx", table_name="products", expressions=["id", "description"], @@ -486,11 +486,11 @@ def test_extract_where_clause(): indexdef = ( "CREATE INDEX products_bm25_idx ON public.products " - "USING bm25 (id, description) WITH (key_field='id') WHERE (rating > 3)" + "USING paradedb (id, description) WITH (key_field='id') WHERE (rating > 3)" ) assert _extract_where_clause(indexdef) == "rating > 3" indexdef_no_where = ( - "CREATE INDEX products_bm25_idx ON public.products USING bm25 (id, description) WITH (key_field='id')" + "CREATE INDEX products_bm25_idx ON public.products USING paradedb (id, description) WITH (key_field='id')" ) assert _extract_where_clause(indexdef_no_where) is None diff --git a/tests/unit/test_indexing_unit.py b/tests/unit/test_indexing_unit.py index 4edda8b..1ee39f1 100644 --- a/tests/unit/test_indexing_unit.py +++ b/tests/unit/test_indexing_unit.py @@ -9,15 +9,16 @@ from paradedb import Tokenizer from paradedb.sqlalchemy.indexing import ( - BM25Field, + ParadeDBField, IndexMeta, _extract_alias, - _extract_bm25_field_list, + _extract_paradedb_field_list, _extract_field_name, _extract_key_field, _extract_tokenizer_name, + _is_paradedb_index, assert_indexed, - validate_bm25_index, + validate_paradedb_index, ) from paradedb.sqlalchemy import tokenizer from paradedb.sqlalchemy.errors import FieldNotIndexedError, InvalidArgumentError @@ -93,251 +94,269 @@ def test_tokenizer_invalid_argument_types(): def test_bm25_index_compile_with_tokenizers(): idx = Index( "products_bm25_idx", - BM25Field(products.c.id), - BM25Field( + ParadeDBField(products.c.id), + ParadeDBField( products.c.description, tokenizer=tokenizer.unicode_words(options={"lowercase": True, "stemmer": "english"}), ), - BM25Field(products.c.category, tokenizer=tokenizer.literal_normalized(options={"alias": "category_exact"})), - postgresql_using="bm25", + ParadeDBField(products.c.category, tokenizer=tokenizer.literal_normalized(options={"alias": "category_exact"})), + postgresql_using="paradedb", postgresql_with={"key_field": "id"}, ) assert ( _sql(CreateIndex(idx).compile(dialect=postgresql.dialect(), compile_kwargs={"literal_binds": True})) == """\ -CREATE INDEX products_bm25_idx ON products USING bm25 (id, ((description)::pdb.unicode_words('lowercase=true','stemmer=english')), ((category)::pdb.literal_normalized('alias=category_exact'))) WITH (key_field = id)""" +CREATE INDEX products_bm25_idx ON products USING paradedb (id, ((description)::pdb.unicode_words('lowercase=true','stemmer=english')), ((category)::pdb.literal_normalized('alias=category_exact'))) WITH (key_field = id)""" ) def test_bm25_index_compile_unicode_omits_none_options(): idx = Index( "products_bm25_idx", - BM25Field(products.c.id), - BM25Field(products.c.description, tokenizer=tokenizer.unicode_words(options={"lowercase": True})), - postgresql_using="bm25", + ParadeDBField(products.c.id), + ParadeDBField(products.c.description, tokenizer=tokenizer.unicode_words(options={"lowercase": True})), + postgresql_using="paradedb", postgresql_with={"key_field": "id"}, ) assert ( _sql(CreateIndex(idx).compile(dialect=postgresql.dialect(), compile_kwargs={"literal_binds": True})) == """\ -CREATE INDEX products_bm25_idx ON products USING bm25 (id, ((description)::pdb.unicode_words('lowercase=true'))) WITH (key_field = id)""" +CREATE INDEX products_bm25_idx ON products USING paradedb (id, ((description)::pdb.unicode_words('lowercase=true'))) WITH (key_field = id)""" ) def test_bm25_index_compile_with_structured_tokenizer_config(): idx = Index( "products_bm25_structured_idx", - BM25Field(products.c.id), - BM25Field( + ParadeDBField(products.c.id), + ParadeDBField( products.c.description, tokenizer=tokenizer.simple( options={"alias": "description_simple", "lowercase": True, "stemmer": "english"} ), ), - postgresql_using="bm25", + postgresql_using="paradedb", postgresql_with={"key_field": "id"}, ) assert ( _sql(CreateIndex(idx).compile(dialect=postgresql.dialect(), compile_kwargs={"literal_binds": True})) == """\ -CREATE INDEX products_bm25_structured_idx ON products USING bm25 (id, ((description)::pdb.simple('alias=description_simple','lowercase=true','stemmer=english'))) WITH (key_field = id)""" +CREATE INDEX products_bm25_structured_idx ON products USING paradedb (id, ((description)::pdb.simple('alias=description_simple','lowercase=true','stemmer=english'))) WITH (key_field = id)""" ) def test_bm25_index_compile_with_tokenizer_positional_and_named_args(): idx = Index( "products_bm25_ngram_idx", - BM25Field(products.c.id), - BM25Field( + ParadeDBField(products.c.id), + ParadeDBField( products.c.description, tokenizer=tokenizer.ngram( 3, 8, options={"alias": "description_ngram", "prefix_only": True, "positions": True} ), ), - postgresql_using="bm25", + postgresql_using="paradedb", postgresql_with={"key_field": "id"}, ) assert ( _sql(CreateIndex(idx).compile(dialect=postgresql.dialect(), compile_kwargs={"literal_binds": True})) == """\ -CREATE INDEX products_bm25_ngram_idx ON products USING bm25 (id, ((description)::pdb.ngram(3,8,'alias=description_ngram','prefix_only=true','positions=true'))) WITH (key_field = id)""" +CREATE INDEX products_bm25_ngram_idx ON products USING paradedb (id, ((description)::pdb.ngram(3,8,'alias=description_ngram','prefix_only=true','positions=true'))) WITH (key_field = id)""" ) def test_bm25_index_compile_lindera_wrapper(): idx = Index( "products_bm25_lindera_idx", - BM25Field(products.c.id), - BM25Field(products.c.description, tokenizer=tokenizer.lindera("japanese", options={"alias": "description_jp"})), - postgresql_using="bm25", + ParadeDBField(products.c.id), + ParadeDBField( + products.c.description, tokenizer=tokenizer.lindera("japanese", options={"alias": "description_jp"}) + ), + postgresql_using="paradedb", postgresql_with={"key_field": "id"}, ) assert ( _sql(CreateIndex(idx).compile(dialect=postgresql.dialect(), compile_kwargs={"literal_binds": True})) == """\ -CREATE INDEX products_bm25_lindera_idx ON products USING bm25 (id, ((description)::pdb.lindera('japanese','alias=description_jp'))) WITH (key_field = id)""" +CREATE INDEX products_bm25_lindera_idx ON products USING paradedb (id, ((description)::pdb.lindera('japanese','alias=description_jp'))) WITH (key_field = id)""" ) def test_bm25_index_compile_regex_pattern_wrapper(): idx = Index( "products_bm25_regex_idx", - BM25Field(products.c.id), - BM25Field( + ParadeDBField(products.c.id), + ParadeDBField( products.c.description, tokenizer=tokenizer.regex_pattern(r"(?i)\\bh\\w*", options={"alias": "description_regex"}), ), - postgresql_using="bm25", + postgresql_using="paradedb", postgresql_with={"key_field": "id"}, ) assert ( _sql(CreateIndex(idx).compile(dialect=postgresql.dialect(), compile_kwargs={"literal_binds": True})) == """\ -CREATE INDEX products_bm25_regex_idx ON products USING bm25 (id, ((description)::pdb.regex_pattern('(?i)\\\\bh\\\\w*','alias=description_regex'))) WITH (key_field = id)""" +CREATE INDEX products_bm25_regex_idx ON products USING paradedb (id, ((description)::pdb.regex_pattern('(?i)\\\\bh\\\\w*','alias=description_regex'))) WITH (key_field = id)""" ) def test_bm25_index_compile_json_key_with_tokenizer(): idx = Index( "products_bm25_json_idx", - BM25Field(products.c.id), - BM25Field( + ParadeDBField(products.c.id), + ParadeDBField( json_text(products.c.metadata, "color"), tokenizer=tokenizer.literal(options={"alias": "metadata_color"}), ), - postgresql_using="bm25", + postgresql_using="paradedb", postgresql_with={"key_field": "id"}, ) assert ( _sql(CreateIndex(idx).compile(dialect=postgresql.dialect(), compile_kwargs={"literal_binds": True})) == """\ -CREATE INDEX products_bm25_json_idx ON products USING bm25 (id, ((metadata ->> 'color')::pdb.literal('alias=metadata_color'))) WITH (key_field = id)""" +CREATE INDEX products_bm25_json_idx ON products USING paradedb (id, ((metadata ->> 'color')::pdb.literal('alias=metadata_color'))) WITH (key_field = id)""" ) def test_bm25_index_compile_multiple_json_keys(): idx = Index( "products_bm25_json_multi_idx", - BM25Field(products.c.id), - BM25Field( + ParadeDBField(products.c.id), + ParadeDBField( json_text(products.c.metadata, "color"), tokenizer=tokenizer.literal(options={"alias": "metadata_color"}), ), - BM25Field( + ParadeDBField( json_text(products.c.metadata, "location"), tokenizer=tokenizer.literal(options={"alias": "metadata_location"}), ), - postgresql_using="bm25", + postgresql_using="paradedb", postgresql_with={"key_field": "id"}, ) assert ( _sql(CreateIndex(idx).compile(dialect=postgresql.dialect(), compile_kwargs={"literal_binds": True})) == """\ -CREATE INDEX products_bm25_json_multi_idx ON products USING bm25 (id, ((metadata ->> 'color')::pdb.literal('alias=metadata_color')), ((metadata ->> 'location')::pdb.literal('alias=metadata_location'))) WITH (key_field = id)""" +CREATE INDEX products_bm25_json_multi_idx ON products USING paradedb (id, ((metadata ->> 'color')::pdb.literal('alias=metadata_color')), ((metadata ->> 'location')::pdb.literal('alias=metadata_location'))) WITH (key_field = id)""" ) def test_bm25_index_compile_non_text_expression_with_pdb_alias(): idx = Index( "products_bm25_expr_idx", - BM25Field(products.c.id), - BM25Field(products.c.description), - BM25Field(pdb.alias(products.c.id + 1, "next_id")), - postgresql_using="bm25", + ParadeDBField(products.c.id), + ParadeDBField(products.c.description), + ParadeDBField(pdb.alias(products.c.id + 1, "next_id")), + postgresql_using="paradedb", postgresql_with={"key_field": "id"}, ) assert ( _sql(CreateIndex(idx).compile(dialect=postgresql.dialect(), compile_kwargs={"literal_binds": True})) == """\ -CREATE INDEX products_bm25_expr_idx ON products USING bm25 (id, description, ((id + 1)::pdb.alias('next_id'))) WITH (key_field = id)""" +CREATE INDEX products_bm25_expr_idx ON products USING paradedb (id, description, ((id + 1)::pdb.alias('next_id'))) WITH (key_field = id)""" ) -def test_bm25_field_non_postgres_compile_raises(): - with pytest.raises(CompileError, match="BM25Field is only supported"): - str(BM25Field(products.c.id).compile(dialect=sqlite.dialect())) +def test_paradedb_field_non_postgres_compile_raises(): + with pytest.raises(CompileError, match="ParadeDBField is only supported"): + str(ParadeDBField(products.c.id).compile(dialect=sqlite.dialect())) + + +def test_is_paradedb_index_only_accepts_paradedb(): + idx = Index( + "products_paradedb_recognition_idx", + ParadeDBField(products.c.id), + postgresql_using="paradedb", + postgresql_with={"key_field": "id"}, + ) + assert _is_paradedb_index(idx) + + for am in ("bm25", "gin", "gist", "btree", None): + other = Index(f"products_{am or 'default'}_idx", products.c.description, postgresql_using=am) + assert not _is_paradedb_index(other), am def test_duplicate_alias_validation_raises(): idx = Index( "products_bm25_alias_idx", - BM25Field(products.c.id), - BM25Field(products.c.description, tokenizer=tokenizer.unicode_words(options={"alias": "description_alias"})), - BM25Field(products.c.category, tokenizer=tokenizer.literal(options={"alias": "description_alias"})), - postgresql_using="bm25", + ParadeDBField(products.c.id), + ParadeDBField( + products.c.description, tokenizer=tokenizer.unicode_words(options={"alias": "description_alias"}) + ), + ParadeDBField(products.c.category, tokenizer=tokenizer.literal(options={"alias": "description_alias"})), + postgresql_using="paradedb", postgresql_with={"key_field": "id"}, ) with pytest.raises(ValueError, match="Duplicate tokenizer alias"): - validate_bm25_index(idx) + validate_paradedb_index(idx) def test_key_field_validation_raises_when_missing(): idx = Index( "products_bm25_missing_key_idx", - BM25Field(products.c.id), - BM25Field(products.c.description), - postgresql_using="bm25", + ParadeDBField(products.c.id), + ParadeDBField(products.c.description), + postgresql_using="paradedb", ) with pytest.raises(ValueError, match="key_field"): - validate_bm25_index(idx) + validate_paradedb_index(idx) def test_key_field_must_exist_in_fields(): idx = Index( "products_bm25_bad_key_idx", - BM25Field(products.c.id), - BM25Field(products.c.description), - postgresql_using="bm25", + ParadeDBField(products.c.id), + ParadeDBField(products.c.description), + postgresql_using="paradedb", postgresql_with={"key_field": "missing"}, ) with pytest.raises(ValueError, match="must match one of the indexed"): - validate_bm25_index(idx) + validate_paradedb_index(idx) def test_key_field_must_be_first_field(): idx = Index( "products_bm25_key_not_first_idx", - BM25Field(products.c.description), - BM25Field(products.c.id), - postgresql_using="bm25", + ParadeDBField(products.c.description), + ParadeDBField(products.c.id), + postgresql_using="paradedb", postgresql_with={"key_field": "id"}, ) - with pytest.raises(ValueError, match="must be the first indexed BM25Field"): - validate_bm25_index(idx) + with pytest.raises(ValueError, match="must be the first indexed ParadeDBField"): + validate_paradedb_index(idx) def test_key_field_must_be_untokenized(): idx = Index( "products_bm25_key_tokenized_idx", - BM25Field(products.c.id, tokenizer=tokenizer.literal(options={"alias": "id_alias"})), - BM25Field(products.c.description), - postgresql_using="bm25", + ParadeDBField(products.c.id, tokenizer=tokenizer.literal(options={"alias": "id_alias"})), + ParadeDBField(products.c.description), + postgresql_using="paradedb", postgresql_with={"key_field": "id"}, ) with pytest.raises(ValueError, match="must be untokenized"): - validate_bm25_index(idx) + validate_paradedb_index(idx) def test_extract_key_field_handles_normalized_indexdef(): - indexdef = "CREATE INDEX idx ON public.products USING bm25 (id, description) WITH (key_field=id)" + indexdef = "CREATE INDEX idx ON public.products USING paradedb (id, description) WITH (key_field=id)" assert _extract_key_field(indexdef) == "id" -def test_extract_bm25_field_list_parses_tokenizer_casts(): +def test_extract_paradedb_field_list_parses_tokenizer_casts(): indexdef = ( - "CREATE INDEX idx ON public.products USING bm25 " + "CREATE INDEX idx ON public.products USING paradedb " "(id, ((description)::pdb.unicode_words('lowercase=true')), " "((category)::pdb.literal_normalized('alias=category_exact'))) WITH (key_field=id)" ) - parts = _extract_bm25_field_list(indexdef) + parts = _extract_paradedb_field_list(indexdef) assert parts == [ "id", "((description)::pdb.unicode_words('lowercase=true'))", @@ -349,6 +368,11 @@ def test_extract_bm25_field_list_parses_tokenizer_casts(): assert _extract_alias(parts[2]) == "category_exact" +def test_extract_paradedb_field_list_parses_legacy_bm25_indexdef(): + indexdef = "CREATE INDEX idx ON public.products USING bm25 (id, description) WITH (key_field=id)" + assert _extract_paradedb_field_list(indexdef) == ["id", "description"] + + def test_extract_field_name_from_json_key_tokenizer_cast(): expr = "((metadata ->> 'color')::pdb.literal('alias=metadata_color'))" assert _extract_field_name(expr) == "metadata"