Skip to content

Commit 9b78f93

Browse files
isaacvandoisaacvando
committed
consolidate how options are passed in to tokenizers
1 parent cc5e4f1 commit 9b78f93

5 files changed

Lines changed: 57 additions & 46 deletions

File tree

‎paradedb/sqlalchemy/tokenizer.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,14 +29,14 @@ def render(self) -> str:
2929
if self.positional_args is not None:
3030
args = [_render_sql_arg(value) for value in self.positional_args]
3131
if self.options:
32-
rendered_options = ",".join(f"{key}={_render_config_value(value)}" for key, value in self.options)
32+
rendered_options = ",".join(f"{key}={_render_config_value(value)}" for key, value in self.options.items())
3333
args.append(_quote_term(rendered_options))
3434
return f"pdb.{self.name}({','.join(args)})"
3535

3636
def extract_alias(self) -> str | None:
3737
if self.options is None:
3838
return None
39-
for key, val in self.options:
39+
for key, val in self.options.items():
4040
if key == "alias":
4141
return val
4242
return None

‎tests/integration/test_alembic_integration.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -293,7 +293,9 @@ def _metadata_with_tokenized_bm25() -> MetaData:
293293
Index(
294294
_AG_IDX,
295295
BM25Field(t.c.id),
296-
BM25Field(t.c.description, tokenizer=tokenizer.simple(alias="description_simple", filters=["lowercase"])),
296+
BM25Field(
297+
t.c.description, tokenizer=tokenizer.simple(options={"alias": "description_simple", "lowercase": True})
298+
),
297299
postgresql_using="bm25",
298300
postgresql_with={"key_field": "id"},
299301
)

‎tests/integration/test_indexing_integration.py‎

Lines changed: 13 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -112,8 +112,8 @@ def test_bm25_index_with_tokenizers_when_supported(engine):
112112
idx = Index(
113113
index_name,
114114
BM25Field(products.c.id),
115-
BM25Field(products.c.description, tokenizer=tokenizer.unicode(lowercase=True)),
116-
BM25Field(products.c.category, tokenizer=tokenizer.literal_normalized(alias="category_exact")),
115+
BM25Field(products.c.description, tokenizer=tokenizer.unicode_words(options={"lowercase": True})),
116+
BM25Field(products.c.category, tokenizer=tokenizer.literal_normalized(options={"alias": "category_exact"})),
117117
postgresql_using="bm25",
118118
postgresql_with={"key_field": "id"},
119119
)
@@ -161,11 +161,11 @@ def test_bm25_index_json_keys_when_supported(engine):
161161
BM25Field(products.c.id),
162162
BM25Field(
163163
json_text(products.c.metadata, "color"),
164-
tokenizer=tokenizer.literal(alias="metadata_color"),
164+
tokenizer=tokenizer.literal(options={"alias": "metadata_color"}),
165165
),
166166
BM25Field(
167167
json_text(products.c.metadata, "location"),
168-
tokenizer=tokenizer.literal(alias="metadata_location"),
168+
tokenizer=tokenizer.literal(options={"alias": "metadata_location"}),
169169
),
170170
postgresql_using="bm25",
171171
postgresql_with={"key_field": "id"},
@@ -293,8 +293,11 @@ def test_duplicate_tokenizer_alias_is_rejected(engine):
293293
idx = Index(
294294
index_name,
295295
BM25Field(products.c.id),
296-
BM25Field(products.c.description, tokenizer=tokenizer.unicode(alias="desc_alias", lowercase=True)),
297-
BM25Field(products.c.description, tokenizer=tokenizer.literal(alias="desc_alias")),
296+
BM25Field(
297+
products.c.description,
298+
tokenizer=tokenizer.unicode_words(options={"alias": "desc_alias", "lowercase": True}),
299+
),
300+
BM25Field(products.c.description, tokenizer=tokenizer.literal(options={"alias": "desc_alias"})),
298301
postgresql_using="bm25",
299302
postgresql_with={"key_field": "id"},
300303
)
@@ -350,8 +353,8 @@ def test_describe_returns_fields_and_aliases(engine):
350353
idx = Index(
351354
index_name,
352355
BM25Field(products.c.id),
353-
BM25Field(products.c.description, tokenizer=tokenizer.unicode(lowercase=True)),
354-
BM25Field(products.c.category, tokenizer=tokenizer.literal_normalized(alias="category_exact")),
356+
BM25Field(products.c.description, tokenizer=tokenizer.unicode_words(options={"lowercase": True})),
357+
BM25Field(products.c.category, tokenizer=tokenizer.literal_normalized(options={"alias": "category_exact"})),
355358
postgresql_using="bm25",
356359
postgresql_with={"key_field": "id"},
357360
)
@@ -389,7 +392,7 @@ def test_describe_includes_tokenizers(engine):
389392
idx = Index(
390393
index_name,
391394
BM25Field(products.c.id),
392-
BM25Field(products.c.description, tokenizer=tokenizer.unicode(lowercase=True)),
395+
BM25Field(products.c.description, tokenizer=tokenizer.unicode_words(options={"lowercase": True})),
393396
BM25Field(products.c.category, tokenizer=tokenizer.literal()),
394397
postgresql_using="bm25",
395398
postgresql_with={"key_field": "id"},
@@ -430,7 +433,7 @@ def test_describe_and_assert_indexed_for_json_expression_tokenizer(engine):
430433
BM25Field(products.c.id),
431434
BM25Field(
432435
json_text(products.c.metadata, "color"),
433-
tokenizer=tokenizer.literal(alias="metadata_color"),
436+
tokenizer=tokenizer.literal(options={"alias": "metadata_color"}),
434437
),
435438
postgresql_using="bm25",
436439
postgresql_with={"key_field": "id"},

‎tests/integration/test_phase0_modules_integration.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -60,8 +60,8 @@ def test_custom_errors_raised_for_validation(engine):
6060
idx = Index(
6161
index_name,
6262
BM25Field(products.c.id),
63-
BM25Field(products.c.description, tokenizer=tokenizer.unicode(alias="dup")),
64-
BM25Field(products.c.description, tokenizer=tokenizer.literal(alias="dup")),
63+
BM25Field(products.c.description, tokenizer=tokenizer.unicode_words(options={"alias": "dup"})),
64+
BM25Field(products.c.description, tokenizer=tokenizer.literal(options={"alias": "dup"})),
6565
postgresql_using="bm25",
6666
postgresql_with={"key_field": "id"},
6767
)

‎tests/unit/test_indexing_unit.py‎

Lines changed: 37 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -40,38 +40,38 @@ def _sql(sql) -> str:
4040

4141

4242
def test_tokenizer_renderers_cover_public_wrappers():
43-
assert tokenizer.unicode(alias="description_unicode", lowercase=True, stemmer="english").render() == (
44-
"pdb.unicode_words('alias=description_unicode,lowercase=true,stemmer=english')"
45-
)
43+
assert tokenizer.unicode_words(
44+
options={"alias": "description_unicode", "lowercase": True, "stemmer": "english"}
45+
).render() == ("pdb.unicode_words('alias=description_unicode,lowercase=true,stemmer=english')")
4646
assert tokenizer.simple(
47-
alias="description_simple", filters=["lowercase", "stemmer"], stemmer="english"
47+
options={"alias": "description_simple", "lowercase": True, "stemmer": "english"}
4848
).render() == ("pdb.simple('alias=description_simple,lowercase=true,stemmer=english')")
49-
assert tokenizer.whitespace(alias="description_whitespace", named_args={"positions": True}).render() == (
49+
assert tokenizer.whitespace(options={"alias": "description_whitespace", "positions": True}).render() == (
5050
"pdb.whitespace('alias=description_whitespace,positions=true')"
5151
)
52-
assert tokenizer.icu(alias="description_icu", filters=["lowercase"]).render() == (
52+
assert tokenizer.icu(options={"alias": "description_icu", "lowercase": True}).render() == (
5353
"pdb.icu('alias=description_icu,lowercase=true')"
5454
)
55-
assert tokenizer.chinese_compatible(alias="description_cjk").render() == (
55+
assert tokenizer.chinese_compatible(options={"alias": "description_cjk"}).render() == (
5656
"pdb.chinese_compatible('alias=description_cjk')"
5757
)
58-
assert tokenizer.jieba(alias="description_jieba", filters=["lowercase"]).render() == (
58+
assert tokenizer.jieba(options={"alias": "description_jieba", "lowercase": True}).render() == (
5959
"pdb.jieba('alias=description_jieba,lowercase=true')"
6060
)
61-
assert tokenizer.literal(alias="category_literal").render() == "pdb.literal('alias=category_literal')"
62-
assert tokenizer.literal_normalized(alias="category_exact").render() == (
61+
assert tokenizer.literal(options={"alias": "category_literal"}).render() == "pdb.literal('alias=category_literal')"
62+
assert tokenizer.literal_normalized(options={"alias": "category_exact"}).render() == (
6363
"pdb.literal_normalized('alias=category_exact')"
6464
)
65-
assert tokenizer.ngram(alias="description_ngram", min_gram=3, max_gram=8, prefix_only=True).render() == (
65+
assert tokenizer.ngram(3, 8, options={"alias": "description_ngram", "prefix_only": True}).render() == (
6666
"pdb.ngram(3,8,'alias=description_ngram,prefix_only=true')"
6767
)
68-
assert tokenizer.lindera("japanese", alias="description_jp").render() == (
68+
assert tokenizer.lindera("japanese", options={"alias": "description_jp"}).render() == (
6969
"pdb.lindera('japanese','alias=description_jp')"
7070
)
71-
assert tokenizer.regex_pattern(r"(?i)\\bh\\w*", alias="description_regex").render() == (
71+
assert tokenizer.regex_pattern(r"(?i)\\bh\\w*", options={"alias": "description_regex"}).render() == (
7272
"pdb.regex_pattern('(?i)\\\\bh\\\\w*','alias=description_regex')"
7373
)
74-
assert tokenizer.source_code(alias="description_source_code", named_args={"ascii_folding": True}).render() == (
74+
assert tokenizer.source_code(options={"alias": "description_source_code", "ascii_folding": True}).render() == (
7575
"pdb.source_code('alias=description_source_code,ascii_folding=true')"
7676
)
7777

@@ -80,8 +80,11 @@ def test_bm25_index_compile_with_tokenizers():
8080
idx = Index(
8181
"products_bm25_idx",
8282
BM25Field(products.c.id),
83-
BM25Field(products.c.description, tokenizer=tokenizer.unicode(lowercase=True, stemmer="english")),
84-
BM25Field(products.c.category, tokenizer=tokenizer.literal_normalized(alias="category_exact")),
83+
BM25Field(
84+
products.c.description,
85+
tokenizer=tokenizer.unicode_words(options={"lowercase": True, "stemmer": "english"}),
86+
),
87+
BM25Field(products.c.category, tokenizer=tokenizer.literal_normalized(options={"alias": "category_exact"})),
8588
postgresql_using="bm25",
8689
postgresql_with={"key_field": "id"},
8790
)
@@ -97,7 +100,7 @@ def test_bm25_index_compile_unicode_omits_none_options():
97100
idx = Index(
98101
"products_bm25_idx",
99102
BM25Field(products.c.id),
100-
BM25Field(products.c.description, tokenizer=tokenizer.unicode(lowercase=True)),
103+
BM25Field(products.c.description, tokenizer=tokenizer.unicode_words(options={"lowercase": True})),
101104
postgresql_using="bm25",
102105
postgresql_with={"key_field": "id"},
103106
)
@@ -115,7 +118,9 @@ def test_bm25_index_compile_with_structured_tokenizer_config():
115118
BM25Field(products.c.id),
116119
BM25Field(
117120
products.c.description,
118-
tokenizer=tokenizer.simple(filters=["lowercase", "stemmer"], stemmer="english", alias="description_simple"),
121+
tokenizer=tokenizer.simple(
122+
options={"alias": "description_simple", "lowercase": True, "stemmer": "english"}
123+
),
119124
),
120125
postgresql_using="bm25",
121126
postgresql_with={"key_field": "id"},
@@ -134,7 +139,7 @@ def test_bm25_index_compile_with_tokenizer_positional_and_named_args():
134139
BM25Field(
135140
products.c.description,
136141
tokenizer=tokenizer.ngram(
137-
min_gram=3, max_gram=8, named_args={"prefix_only": True, "positions": True}, alias="description_ngram"
142+
3, 8, options={"alias": "description_ngram", "prefix_only": True, "positions": True}
138143
),
139144
),
140145
postgresql_using="bm25",
@@ -151,7 +156,7 @@ def test_bm25_index_compile_lindera_wrapper():
151156
idx = Index(
152157
"products_bm25_lindera_idx",
153158
BM25Field(products.c.id),
154-
BM25Field(products.c.description, tokenizer=tokenizer.lindera("japanese", alias="description_jp")),
159+
BM25Field(products.c.description, tokenizer=tokenizer.lindera("japanese", options={"alias": "description_jp"})),
155160
postgresql_using="bm25",
156161
postgresql_with={"key_field": "id"},
157162
)
@@ -167,7 +172,8 @@ def test_bm25_index_compile_regex_pattern_wrapper():
167172
"products_bm25_regex_idx",
168173
BM25Field(products.c.id),
169174
BM25Field(
170-
products.c.description, tokenizer=tokenizer.regex_pattern(r"(?i)\\bh\\w*", alias="description_regex")
175+
products.c.description,
176+
tokenizer=tokenizer.regex_pattern(r"(?i)\\bh\\w*", options={"alias": "description_regex"}),
171177
),
172178
postgresql_using="bm25",
173179
postgresql_with={"key_field": "id"},
@@ -185,7 +191,7 @@ def test_bm25_index_compile_json_key_with_tokenizer():
185191
BM25Field(products.c.id),
186192
BM25Field(
187193
json_text(products.c.metadata, "color"),
188-
tokenizer=tokenizer.literal(alias="metadata_color"),
194+
tokenizer=tokenizer.literal(options={"alias": "metadata_color"}),
189195
),
190196
postgresql_using="bm25",
191197
postgresql_with={"key_field": "id"},
@@ -203,11 +209,11 @@ def test_bm25_index_compile_multiple_json_keys():
203209
BM25Field(products.c.id),
204210
BM25Field(
205211
json_text(products.c.metadata, "color"),
206-
tokenizer=tokenizer.literal(alias="metadata_color"),
212+
tokenizer=tokenizer.literal(options={"alias": "metadata_color"}),
207213
),
208214
BM25Field(
209215
json_text(products.c.metadata, "location"),
210-
tokenizer=tokenizer.literal(alias="metadata_location"),
216+
tokenizer=tokenizer.literal(options={"alias": "metadata_location"}),
211217
),
212218
postgresql_using="bm25",
213219
postgresql_with={"key_field": "id"},
@@ -245,8 +251,8 @@ def test_duplicate_alias_validation_raises():
245251
idx = Index(
246252
"products_bm25_alias_idx",
247253
BM25Field(products.c.id),
248-
BM25Field(products.c.description, tokenizer=tokenizer.unicode(alias="description_alias")),
249-
BM25Field(products.c.category, tokenizer=tokenizer.literal(alias="description_alias")),
254+
BM25Field(products.c.description, tokenizer=tokenizer.unicode_words(options={"alias": "description_alias"})),
255+
BM25Field(products.c.category, tokenizer=tokenizer.literal(options={"alias": "description_alias"})),
250256
postgresql_using="bm25",
251257
postgresql_with={"key_field": "id"},
252258
)
@@ -296,7 +302,7 @@ def test_key_field_must_be_first_field():
296302
def test_key_field_must_be_untokenized():
297303
idx = Index(
298304
"products_bm25_key_tokenized_idx",
299-
BM25Field(products.c.id, tokenizer=tokenizer.literal(alias="id_alias")),
305+
BM25Field(products.c.id, tokenizer=tokenizer.literal(options={"alias": "id_alias"})),
300306
BM25Field(products.c.description),
301307
postgresql_using="bm25",
302308
postgresql_with={"key_field": "id"},
@@ -314,14 +320,14 @@ def test_extract_key_field_handles_normalized_indexdef():
314320
def test_extract_bm25_field_list_parses_tokenizer_casts():
315321
indexdef = (
316322
"CREATE INDEX idx ON public.products USING bm25 "
317-
"(id, ((description)::pdb.unicode_words(lowercase=true)), "
318-
"((category)::pdb.literal_normalized(alias=category_exact))) WITH (key_field=id)"
323+
"(id, ((description)::pdb.unicode_words('lowercase=true')), "
324+
"((category)::pdb.literal_normalized('alias=category_exact'))) WITH (key_field=id)"
319325
)
320326
parts = _extract_bm25_field_list(indexdef)
321327
assert parts == [
322328
"id",
323-
"((description)::pdb.unicode_words(lowercase=true))",
324-
"((category)::pdb.literal_normalized(alias=category_exact))",
329+
"((description)::pdb.unicode_words('lowercase=true'))",
330+
"((category)::pdb.literal_normalized('alias=category_exact'))",
325331
]
326332
assert _extract_field_name(parts[0]) == "id"
327333
assert _extract_field_name(parts[1]) == "description"

0 commit comments

Comments
 (0)