Skip to content

Commit aebbe1a

Browse files
isaacvandoisaacvando
authored
feat: Allow tokenizers to accept arbitrary options (#49)
1 parent 0959d50 commit aebbe1a

8 files changed

Lines changed: 140 additions & 662 deletions

File tree

‎src/ParadeDB.EntityFrameworkCore/Internal/Query/ParadeDbEvaluatableExpressionFilterPlugin.cs‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
using System.Linq.Expressions;
22
using Microsoft.EntityFrameworkCore.Query;
3+
using ParadeDB.EntityFrameworkCore.Extensions;
34

45
namespace ParadeDB.EntityFrameworkCore.Internal.Query;
56

@@ -20,6 +21,12 @@ or nameof(Pdb.Fuzzy)
2021
or nameof(Pdb.Slop)
2122
&& call.Arguments.Count > 1
2223
)
24+
|| (
25+
call.Method.DeclaringType == typeof(ParadeDbFunctionsExtensions)
26+
&& call.Method.Name
27+
is nameof(ParadeDbFunctionsExtensions.Tokenize)
28+
or nameof(ParadeDbFunctionsExtensions.TokenizeAsArray)
29+
)
2330
)
2431
)
2532
{

‎src/ParadeDB.EntityFrameworkCore/TokenFilter.cs‎

Lines changed: 0 additions & 76 deletions
This file was deleted.

‎src/ParadeDB.EntityFrameworkCore/Tokenizer.cs‎

Lines changed: 51 additions & 80 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,15 @@
11
using System.Collections.Frozen;
2+
using System.Collections.ObjectModel;
23
using System.Diagnostics.CodeAnalysis;
4+
using System.Globalization;
35

46
namespace ParadeDB.EntityFrameworkCore;
57

68
public sealed class Tokenizer
79
{
810
private readonly string _name;
911
private readonly string[] _args;
10-
private readonly TokenFilter[] _filters;
12+
private readonly IReadOnlyDictionary<string, object> _options;
1113

1214
private static readonly FrozenDictionary<LinderaLanguage, string> LinderaLanguageArgs =
1315
new Dictionary<LinderaLanguage, string>
@@ -17,111 +19,80 @@ public sealed class Tokenizer
1719
{ LinderaLanguage.Korean, "korean" },
1820
}.ToFrozenDictionary();
1921

20-
private static readonly FrozenDictionary<TokenChars, string> TokenCharsArgs = new Dictionary<
21-
TokenChars,
22-
string
23-
>
24-
{
25-
{ TokenChars.Letter, "letter" },
26-
{ TokenChars.Digit, "digit" },
27-
{ TokenChars.Whitespace, "whitespace" },
28-
{ TokenChars.Punctuation, "punctuation" },
29-
{ TokenChars.Symbol, "symbol" },
30-
}.ToFrozenDictionary();
31-
32-
private Tokenizer(string name, string[] args, params TokenFilter[] filters)
22+
private Tokenizer(string name, string[] args, Dictionary<string, object> options)
3323
{
3424
_name = name;
3525
_args = args;
36-
_filters = filters;
26+
_options = new ReadOnlyDictionary<string, object>(new Dictionary<string, object>(options));
3727
}
3828

3929
public override string ToString()
4030
{
41-
var args = _args.Concat(_filters.Select(f => f.ToString())).ToList();
31+
var args = _args
32+
.Concat(
33+
_options.Select(option =>
34+
$"'{Quote(option.Key)}={FormatOptionValue(option.Value, option.Key)}'"
35+
)
36+
)
37+
.ToList();
4238

43-
return args.Count == 0 ? $"pdb.{_name}" : $"pdb.{_name}({string.Join(", ", args)})";
39+
return args.Count == 0 ? $"pdb.{_name}" : $"pdb.{_name}({string.Join(",", args)})";
4440
}
4541

46-
public static Tokenizer Unicode(params TokenFilter[] filters) =>
47-
new("unicode_words", [], filters);
42+
private static string FormatOptionValue(object value, string key) =>
43+
value switch
44+
{
45+
bool b => b.ToString().ToLowerInvariant(),
46+
string s => Quote(s),
47+
int n => n.ToString(CultureInfo.InvariantCulture),
48+
float n => n.ToString(CultureInfo.InvariantCulture),
49+
_ => throw new ArgumentException(
50+
$"Tokenizer option '{key}' has unsupported value type '{value?.GetType().Name}'. Each tokenizer option value must be a bool, string, int, or float.",
51+
"options"
52+
),
53+
};
4854

49-
public static Tokenizer Unicode(bool removeEmojis, params TokenFilter[] filters) =>
50-
removeEmojis
51-
? new Tokenizer("unicode_words", ["'remove_emojis=true'"], filters)
52-
: Unicode(filters);
55+
private static string Quote(string value) => value.Replace("'", "''");
5356

54-
public static Tokenizer Literal => new("literal", []);
57+
public static Tokenizer Unicode(Dictionary<string, object> options) =>
58+
new("unicode_words", [], options);
5559

56-
public static Tokenizer LiteralNormalized(params TokenFilter[] filters) =>
57-
new("literal_normalized", [], filters);
60+
public static Tokenizer Literal(Dictionary<string, object> options) =>
61+
new("literal", [], options);
5862

59-
public static Tokenizer Whitespace(params TokenFilter[] filters) =>
60-
new("whitespace", [], filters);
63+
public static Tokenizer LiteralNormalized(Dictionary<string, object> options) =>
64+
new("literal_normalized", [], options);
6165

62-
public static Tokenizer Ngram(int minGram, int maxGram, params TokenFilter[] filters) =>
63-
new("ngram", [$"{minGram},{maxGram}"], filters);
66+
public static Tokenizer Whitespace(Dictionary<string, object> options) =>
67+
new("whitespace", [], options);
6468

65-
public static Tokenizer NgramPrefixOnly(
66-
int minGram,
67-
int maxGram,
68-
params TokenFilter[] filters
69-
) => new("ngram", [$"{minGram}, {maxGram}, 'prefix_only=true'"], filters);
70-
71-
public static Tokenizer NgramPositions(int gramSize, params TokenFilter[] filters) =>
72-
new("ngram", [$"{gramSize}, {gramSize}, 'positions=true'"], filters);
73-
74-
public static Tokenizer EdgeNgram(int minGram, int maxGram, params TokenFilter[] filters) =>
75-
new("edge_ngram", [$"{minGram},{maxGram}"], filters);
69+
public static Tokenizer Ngram(int minGram, int maxGram, Dictionary<string, object> options) =>
70+
new("ngram", [$"{minGram},{maxGram}"], options);
7671

7772
public static Tokenizer EdgeNgram(
7873
int minGram,
7974
int maxGram,
80-
TokenChars tokenChars,
81-
params TokenFilter[] filters
82-
) =>
83-
new(
84-
"edge_ngram",
85-
[$"{minGram},{maxGram}", $"'token_chars={SerializeTokenChars(tokenChars)}'"],
86-
filters
87-
);
75+
Dictionary<string, object> options
76+
) => new("edge_ngram", [$"{minGram},{maxGram}"], options);
8877

89-
public static Tokenizer Simple(params TokenFilter[] filters) => new("simple", [], filters);
78+
public static Tokenizer Simple(Dictionary<string, object> options) =>
79+
new("simple", [], options);
9080

9181
public static Tokenizer RegexPattern(
9282
[StringSyntax(StringSyntaxAttribute.Regex)] string pattern,
93-
params TokenFilter[] filters
94-
) => new("regex_pattern", [$"'{pattern}'"], filters);
95-
96-
public static Tokenizer ChineseCompatible(params TokenFilter[] filters) =>
97-
new("chinese_compatible", [], filters);
98-
99-
public static Tokenizer Lindera(LinderaLanguage language, params TokenFilter[] filters) =>
100-
new("lindera", [LinderaLanguageArgs[language]], filters);
101-
102-
public static Tokenizer Lindera(
103-
LinderaLanguage language,
104-
bool keepWhitespace,
105-
params TokenFilter[] filters
106-
) =>
107-
keepWhitespace
108-
? new Tokenizer(
109-
"lindera",
110-
[$"{LinderaLanguageArgs[language]}, 'keep_whitespace=true'"],
111-
filters
112-
)
113-
: Lindera(language, filters);
83+
Dictionary<string, object> options
84+
) => new("regex_pattern", [$"'{Quote(pattern)}'"], options);
85+
86+
public static Tokenizer ChineseCompatible(Dictionary<string, object> options) =>
87+
new("chinese_compatible", [], options);
11488

115-
public static Tokenizer Icu(params TokenFilter[] filters) => new("icu", [], filters);
89+
public static Tokenizer Lindera(LinderaLanguage language, Dictionary<string, object> options) =>
90+
new("lindera", [$"'{LinderaLanguageArgs[language]}'"], options);
11691

117-
public static Tokenizer Jieba(params TokenFilter[] filters) => new("jieba", [], filters);
92+
public static Tokenizer Icu(Dictionary<string, object> options) => new("icu", [], options);
11893

119-
public static Tokenizer SourceCode(params TokenFilter[] filters) =>
120-
new("source_code", [], filters);
94+
public static Tokenizer Jieba(Dictionary<string, object> options) => new("jieba", [], options);
12195

122-
private static string SerializeTokenChars(TokenChars chars) =>
123-
string.Join(
124-
",",
125-
Enum.GetValues<TokenChars>().Where(c => chars.HasFlag(c)).Select(c => TokenCharsArgs[c])
126-
);
96+
public static Tokenizer SourceCode(Dictionary<string, object> options) =>
97+
new("source_code", [], options);
12798
}

‎tests/ParadeDB.EntityFrameworkCore.IntegrationTests/QueryTests.cs‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -272,6 +272,75 @@ FROM products AS p
272272
await query.ToListAsync();
273273
}
274274

275+
private static readonly (
276+
Tokenizer Tokenizer,
277+
string ExpectedSqlSnippet
278+
)[] TokenizerVariations =
279+
[
280+
(
281+
Tokenizer.Unicode(new() { ["remove_emojis"] = true }),
282+
"::pdb.unicode_words('remove_emojis=true')"
283+
),
284+
(
285+
Tokenizer.Simple(new() { ["stemmer"] = "english", ["alias"] = "simple_description" }),
286+
"::pdb.simple('stemmer=english','alias=simple_description')"
287+
),
288+
(Tokenizer.Icu([]), "::pdb.icu"),
289+
(Tokenizer.ChineseCompatible([]), "::pdb.chinese_compatible"),
290+
(Tokenizer.Jieba([]), "::pdb.jieba"),
291+
(
292+
Tokenizer.Lindera(LinderaLanguage.Chinese, new() { ["keep_whitespace"] = true }),
293+
"::pdb.lindera('chinese','keep_whitespace=true')"
294+
),
295+
(Tokenizer.Literal([]), "::pdb.literal"),
296+
(
297+
Tokenizer.LiteralNormalized(new() { ["trim"] = true }),
298+
"::pdb.literal_normalized('trim=true')"
299+
),
300+
(
301+
Tokenizer.Ngram(3, 3, new() { ["positions"] = true, ["prefix_only"] = true }),
302+
"::pdb.ngram(3,3,'positions=true','prefix_only=true')"
303+
),
304+
(
305+
Tokenizer.EdgeNgram(2, 5, new() { ["token_chars"] = "letter,digit,punctuation" }),
306+
"::pdb.edge_ngram(2,5,'token_chars=letter,digit,punctuation')"
307+
),
308+
(Tokenizer.RegexPattern("[a-z]+", []), "::pdb.regex_pattern('[a-z]+')"),
309+
(Tokenizer.SourceCode([]), "::pdb.source_code"),
310+
(Tokenizer.Whitespace([]), "::pdb.whitespace"),
311+
(
312+
Tokenizer.Whitespace(new() { ["alias"] = "'escape me'" }),
313+
"::pdb.whitespace('alias=''escape me''')"
314+
),
315+
];
316+
317+
[Test]
318+
public async Task MatchAll_WithTokenizerVariations_ExecutesSuccessfully()
319+
{
320+
await using var context = DbFixture.CreateContext();
321+
322+
foreach (var tokenizerVariation in TokenizerVariations)
323+
{
324+
var query = context
325+
.Products.Where(p =>
326+
EF.Functions.MatchAll(
327+
p.Description,
328+
EF.Functions.Tokenize("running shoes", tokenizerVariation.Tokenizer)
329+
)
330+
)
331+
.Select(p => new { p.Id, p.Description });
332+
333+
var sql = $$"""
334+
SELECT p.id AS "Id", p.description AS "Description"
335+
FROM products AS p
336+
WHERE p.description &&& 'running shoes'{{tokenizerVariation.ExpectedSqlSnippet}}
337+
""";
338+
339+
AssertSql(query, sql);
340+
await query.ToListAsync();
341+
}
342+
}
343+
275344
[Test]
276345
public async Task MatchAny_ExecutesSuccessfully()
277346
{

0 commit comments

Comments
 (0)