Add support for search tokenizers

Author: isaacvando

src/Extensions/ParadeDbIndexBuilderExtensions.cs

@@ -138,6 +138,16 @@ public Bm25IndexBuilder<TEntity> IsCreatedConcurrently(bool createdConcurrently
         return this;
     }
 
+    public Bm25IndexBuilder<TEntity> HasSearchTokenizer(Tokenizer tokenizer)
+    {
+        _indexBuilder.HasAnnotation(
+            ParadeDbAnnotationNames.Bm25SearchTokenizer,
+            tokenizer.ToSearchString()
+        );
+
+        return this;
+    }
+
     private void AddField(string field, string kind, Tokenizer? tokenizer, string? @alias)
     {
         var properties = GetAnnotation(ParadeDbAnnotationNames.Bm25FieldProperties);

src/Internal/Metadata/ParadeDbAnnotationNames.cs

@@ -7,6 +7,7 @@ internal static class ParadeDbAnnotationNames
     public const string Bm25FieldKinds = "ParadeDB:Bm25FieldKinds";
     public const string Bm25FieldTokenizers = "ParadeDB:Bm25FieldTokenizers";
     public const string Bm25FieldAliases = "ParadeDB:Bm25FieldAliases";
+    public const string Bm25SearchTokenizer = "ParadeDB:Bm25SearchTokenizer";
     public const string Bm25KeyField = "ParadeDB:Bm25KeyField";
     public const string Bm25Fields = "ParadeDB:Bm25Fields";
 }

src/Internal/Metadata/ParadeDbAnnotationProvider.cs

@@ -81,6 +81,17 @@ is not string keyPropertyName
                 )
                 .ToArray()
         );
+
+        if (
+            mappedIndex.FindAnnotation(ParadeDbAnnotationNames.Bm25SearchTokenizer)?.Value
+            is string searchTokenizer
+        )
+        {
+            yield return new Annotation(
+                ParadeDbAnnotationNames.Bm25SearchTokenizer,
+                searchTokenizer
+            );
+        }
     }
 
     private static string RenderField(

src/Internal/Migrations/ParadeDbMigrationsSqlGenerator.cs

@@ -35,8 +35,11 @@ is not string keyField
         }
 
         var helper = Dependencies.SqlGenerationHelper;
+        var stringMapping = Dependencies.TypeMappingSource.FindMapping(typeof(string))!;
         var createdConcurrently =
             operation.FindAnnotation(NpgsqlAnnotationNames.CreatedConcurrently)?.Value is true;
+        var searchTokenizer =
+            operation.FindAnnotation(ParadeDbAnnotationNames.Bm25SearchTokenizer)?.Value as string;
 
         builder
             .Append("CREATE INDEX ")
@@ -47,12 +50,16 @@ is not string keyField
             .Append(" USING bm25 (")
             .Append(string.Join(", ", fields))
             .Append(") WITH (key_field = ")
-            .Append(
-                Dependencies
-                    .TypeMappingSource.FindMapping(typeof(string))!
-                    .GenerateSqlLiteral(keyField)
-            )
-            .Append(")");
+            .Append(stringMapping.GenerateSqlLiteral(keyField));
+
+        if (searchTokenizer is not null)
+        {
+            builder
+                .Append(", search_tokenizer = ")
+                .Append(stringMapping.GenerateSqlLiteral(searchTokenizer));
+        }
+
+        builder.Append(")");
 
         if (operation.Filter is not null)
         {

src/Tokenizer.cs

@@ -41,6 +41,19 @@ public override string ToString()
         return args.Count == 0 ? $"pdb.{_name}" : $"pdb.{_name}({string.Join(",", args)})";
     }
 
+    internal string ToSearchString()
+    {
+        var args = _args.Concat(
+            _options.Select(option =>
+                $"{option.Key}={FormatSearchOptionValue(option.Value, option.Key)}"
+            )
+        );
+
+        var argsArray = args as string[] ?? args.ToArray();
+
+        return argsArray.Length == 0 ? _name : $"{_name}({string.Join(",", argsArray)})";
+    }
+
     private static string FormatOptionValue(object value, string key) =>
         value switch
         {
@@ -54,6 +67,19 @@ private static string FormatOptionValue(object value, string key) =>
             ),
         };
 
+    private static string FormatSearchOptionValue(object value, string key) =>
+        value switch
+        {
+            bool b => b.ToString().ToLowerInvariant(),
+            string s => s,
+            int n => n.ToString(CultureInfo.InvariantCulture),
+            float n => n.ToString(CultureInfo.InvariantCulture),
+            _ => throw new ArgumentException(
+                $"Tokenizer option '{key}' has unsupported value type '{value?.GetType().Name}'. Each tokenizer option value must be a bool, string, int, or float.",
+                "options"
+            ),
+        };
+
     private static string Quote(string value) => value.Replace("'", "''");
 
     public static Tokenizer Unicode(Dictionary<string, object>? options = null) =>

tests/IndexingTest.cs

@@ -80,7 +80,8 @@ protected override void OnModelCreating(ModelBuilder modelBuilder)
                     )
                     .HasField(e => e.Rating, new FieldAlias("my_rating_alias"))
                     .HasField("rating + 1", new FieldAlias("escape' me"))
-                    .HasFilter("rating > 0");
+                    .HasFilter("rating > 0")
+                    .HasSearchTokenizer(Tokenizer.Simple(new() { ["lowercase"] = false }));
             });
         }
     }
@@ -105,7 +106,7 @@ rating int
 
         sql.ShouldBe(
             """
-            CREATE INDEX indexing_items_idx ON indexing_items USING bm25 (id, (description::pdb.ngram(3,3,'positions=true')), ((metadata ->> 'color')::pdb.literal('alias=metadata_color')), (rating::pdb.alias('my_rating_alias')), ((rating + 1)::pdb.alias('escape'' me'))) WITH (key_field = 'id') WHERE rating > 0;
+            CREATE INDEX indexing_items_idx ON indexing_items USING bm25 (id, (description::pdb.ngram(3,3,'positions=true')), ((metadata ->> 'color')::pdb.literal('alias=metadata_color')), (rating::pdb.alias('my_rating_alias')), ((rating + 1)::pdb.alias('escape'' me'))) WITH (key_field = 'id', search_tokenizer = 'simple(lowercase=false)') WHERE rating > 0;
 
             """
         );