Ability to control which fields are full text indexed

ejsmith · ejsmith · commit 4b2d89a0c6ed · 2025-11-18T23:06:56.000-06:00
diff --git a/src/Foundatio.Parsers.SqlQueries/Extensions/SqlNodeExtensions.cs b/src/Foundatio.Parsers.SqlQueries/Extensions/SqlNodeExtensions.cs
@@ -187,7 +187,7 @@ public static string ToDynamicLinqString(this TermNode node, ISqlQueryVisitorCon
                         builder.Append(i.IsFirst ? "(" : " OR ");
                         builder.Append(fieldPrefix);
 
-                        if (context.FullTextSearchEnabled)
+                        if (context.FullTextFields.Contains(kvp.Key.Name, StringComparer.OrdinalIgnoreCase))
                         {
                             builder.Append("FTS.Contains(");
                             builder.Append(kvp.Key.Name);
@@ -215,12 +215,12 @@ public static string ToDynamicLinqString(this TermNode node, ISqlQueryVisitorCon
                         builder.Append(i.IsFirst ? "(" : " OR ");
                         builder.Append(fieldPrefix);
 
-                        if (context.FullTextSearchEnabled)
+                        if (context.FullTextFields.Contains(kvp.Key.Name, StringComparer.OrdinalIgnoreCase))
                         {
                             builder.Append("FTS.Contains(");
                             builder.Append(kvp.Key.Name);
                             builder.Append(", ");
-                            AppendField(builder, kvp.Key, token + "*", context);
+                            AppendField(builder, kvp.Key, "\\\"" + token + "*\\\"", context);
                             builder.Append(")");
                         }
                         else
@@ -266,19 +266,45 @@ public static string ToDynamicLinqString(this TermNode node, ISqlQueryVisitorCon
         else if (searchOperator == SqlSearchOperator.Contains)
         {
             builder.Append(fieldPrefix);
-            builder.Append(field.Name);
-            builder.Append(".Contains(");
-            AppendField(builder, field, node.Term, context);
-            builder.Append(")");
+
+            if (context.FullTextFields.Contains(field.Name, StringComparer.OrdinalIgnoreCase))
+            {
+                builder.Append("FTS.Contains(");
+                builder.Append(field.Name);
+                builder.Append(", ");
+                AppendField(builder, field, node.Term, context);
+                builder.Append(")");
+            }
+            else
+            {
+                builder.Append(field.Name);
+                builder.Append(".Contains(");
+                AppendField(builder, field, node.Term, context);
+                builder.Append(")");
+            }
+
             builder.Append(fieldSuffix);
         }
         else
         {
             builder.Append(fieldPrefix);
-            builder.Append(field.Name);
-            builder.Append(".StartsWith(");
-            AppendField(builder, field, node.Term, context);
-            builder.Append(")");
+
+            if (context.FullTextFields.Contains(field.Name, StringComparer.OrdinalIgnoreCase))
+            {
+                builder.Append("FTS.Contains(");
+                builder.Append(field.Name);
+                builder.Append(", ");
+                AppendField(builder, field, "\\\"" + node.Term + "*\\\"", context);
+                builder.Append(")");
+            }
+            else
+            {
+                builder.Append(field.Name);
+                builder.Append(".Contains(");
+                AppendField(builder, field, node.Term, context);
+                builder.Append(")");
+            }
+
             builder.Append(fieldSuffix);
         }
 
diff --git a/src/Foundatio.Parsers.SqlQueries/SqlQueryParser.cs b/src/Foundatio.Parsers.SqlQueries/SqlQueryParser.cs
@@ -225,7 +225,7 @@ private void SetupQueryVisitorContextDefaults(IQueryVisitorContext context)
             sqlContext.DateTimeParser = Configuration.DateTimeParser;
             sqlContext.DateOnlyParser = Configuration.DateOnlyParser;
             sqlContext.DefaultSearchOperator = Configuration.DefaultFieldsSearchOperator;
-            sqlContext.FullTextSearchEnabled = Configuration.FullTextSearchEnabled;
+            sqlContext.FullTextFields = Configuration.FullTextFields;
         }
     }
 }
diff --git a/src/Foundatio.Parsers.SqlQueries/SqlQueryParserConfiguration.cs b/src/Foundatio.Parsers.SqlQueries/SqlQueryParserConfiguration.cs
@@ -24,8 +24,7 @@ public SqlQueryParserConfiguration()
     public ILoggerFactory LoggerFactory { get; private set; } = NullLoggerFactory.Instance;
     public string[] DefaultFields { get; private set; }
     public SqlSearchOperator DefaultFieldsSearchOperator { get; private set; } = SqlSearchOperator.StartsWith;
-    public bool FullTextSearchEnabled { get; private set; } = false;
-
+    public string[] FullTextFields { get; private set; }
     public int MaxFieldDepth { get; private set; } = 10;
     public QueryFieldResolver FieldResolver { get; private set; }
     public Action<SearchTerm> SearchTokenizer { get; set; } = static _ => { };
@@ -61,9 +60,9 @@ public SqlQueryParserConfiguration SetSearchTokenizer(Action<SearchTerm> tokeniz
         return this;
     }
 
-    public SqlQueryParserConfiguration UseFullTextSearch(bool useFullTextSearch = true)
+    public SqlQueryParserConfiguration SetFullTextFields(string[] fields)
     {
-        FullTextSearchEnabled = useFullTextSearch;
+        FullTextFields = fields;
         return this;
     }
 
diff --git a/src/Foundatio.Parsers.SqlQueries/Visitors/ISqlQueryVisitorContext.cs b/src/Foundatio.Parsers.SqlQueries/Visitors/ISqlQueryVisitorContext.cs
@@ -8,7 +8,7 @@ public interface ISqlQueryVisitorContext : IQueryVisitorContext
 {
     List<EntityFieldInfo> Fields { get; set; }
     SqlSearchOperator DefaultSearchOperator { get; set; }
-    bool FullTextSearchEnabled { get; set; }
+    string[] FullTextFields { get; set; }
     Action<SearchTerm> SearchTokenizer { get; set; }
     Func<string, string> DateTimeParser { get; set; }
     Func<string, string> DateOnlyParser { get; set; }
diff --git a/src/Foundatio.Parsers.SqlQueries/Visitors/SqlQueryVisitorContext.cs b/src/Foundatio.Parsers.SqlQueries/Visitors/SqlQueryVisitorContext.cs
@@ -11,7 +11,7 @@ public class SqlQueryVisitorContext : QueryVisitorContext, ISqlQueryVisitorConte
 {
     public List<EntityFieldInfo> Fields { get; set; }
     public SqlSearchOperator DefaultSearchOperator { get; set; } = SqlSearchOperator.StartsWith;
-    public bool FullTextSearchEnabled { get; set; } = false;
+    public string[] FullTextFields { get; set; } = [];
     public Action<SearchTerm> SearchTokenizer { get; set; } = static _ => { };
     public Func<string, string> DateTimeParser { get; set; }
     public Func<string, string> DateOnlyParser { get; set; }
diff --git a/tests/Foundatio.Parsers.SqlQueries.Tests/SqlQueryParserTests.cs b/tests/Foundatio.Parsers.SqlQueries.Tests/SqlQueryParserTests.cs
@@ -7,6 +7,7 @@
 using Foundatio.Parsers.LuceneQueries.Visitors;
 using Foundatio.Parsers.SqlQueries.Visitors;
 using Foundatio.Xunit;
+using Microsoft.Data.SqlClient;
 using Microsoft.EntityFrameworkCore;
 using Microsoft.EntityFrameworkCore.Infrastructure;
 using Microsoft.Extensions.DependencyInjection;
@@ -70,14 +71,17 @@ public async Task CanSearchDefaultFields()
 
         var context = parser.GetContext(db.Employees.EntityType);
 
-        string sqlExpected = db.Employees.Where(e => e.FullName.StartsWith("John") || e.Title.StartsWith("John")).ToQueryString();
-        string sqlActual = db.Employees.Where("""FullName.StartsWith("John") || Title.StartsWith("John") """).ToQueryString();
+        string sqlExpected = db.Employees.Where(e => EF.Functions.Contains(e.FullName, "\"John*\"") || EF.Functions.Contains(e.Title, "\"John*\"")).ToQueryString();
+        string sqlActual = db.Employees.Where(parser.ParsingConfig, """FTS.Contains(FullName, "\"John*\"") || FTS.Contains(Title, "\"John*\"") """).ToQueryString();
         Assert.Equal(sqlExpected, sqlActual);
         string sql = await parser.ToDynamicLinqAsync("John", context);
-        sqlActual = db.Employees.Where(sql).ToQueryString();
-        var results = await db.Employees.Where(sql).ToListAsync();
-        Assert.Single(results);
+        sqlActual = db.Employees.Where(parser.ParsingConfig, sql).ToQueryString();
         Assert.Equal(sqlExpected, sqlActual);
+
+        await WaitForFullTextIndexAsync(db, "ftCatalog");
+
+        var results = await db.Employees.Where(parser.ParsingConfig, sql).ToListAsync();
+        Assert.Single(results);
     }
 
     [Fact]
@@ -101,28 +105,31 @@ public async Task CanSearchWithTokenizer()
 
         var context = parser.GetContext(db.Employees.EntityType);
 
-        string sqlExpected = db.Employees.Where(e => e.NationalPhoneNumber.StartsWith("2142222222")).ToQueryString();
-        string sqlActual = db.Employees.Where("NationalPhoneNumber.StartsWith(\"2142222222\")").ToQueryString();
+        string sqlExpected = db.Employees.Where(e => EF.Functions.Contains(e.NationalPhoneNumber, "\"2142222222*\"")).ToQueryString();
+        string sqlActual = db.Employees.Where(parser.ParsingConfig, "FTS.Contains(NationalPhoneNumber, \"\\\"2142222222*\\\"\")").ToQueryString();
         Assert.Equal(sqlExpected, sqlActual);
 
         string sql = await parser.ToDynamicLinqAsync("214-222-2222", context);
         _logger.LogInformation(sql);
-        sqlActual = db.Employees.Where(sql).ToQueryString();
-        var results = await db.Employees.Where(sql).ToListAsync();
-        Assert.Single(results);
         Assert.Equal(sqlExpected, sqlActual);
 
+        await WaitForFullTextIndexAsync(db, "ftCatalog");
+
+        sqlActual = db.Employees.Where(parser.ParsingConfig, sql).ToQueryString();
+        var results = await db.Employees.Where(parser.ParsingConfig, sql).ToListAsync();
+        Assert.Single(results);
+
         sql = await parser.ToDynamicLinqAsync("2142222222", context);
         _logger.LogInformation(sql);
-        sqlActual = db.Employees.Where(sql).ToQueryString();
-        results = await db.Employees.Where(sql).ToListAsync();
+        sqlActual = db.Employees.Where(parser.ParsingConfig, sql).ToQueryString();
+        results = await db.Employees.Where(parser.ParsingConfig, sql).ToListAsync();
         Assert.Single(results);
         Assert.Equal(sqlExpected, sqlActual);
 
         sql = await parser.ToDynamicLinqAsync("21422", context);
         _logger.LogInformation(sql);
-        sqlActual = db.Employees.Where(sql).ToQueryString();
-        results = await db.Employees.Where(sql).ToListAsync();
+        sqlActual = db.Employees.Where(parser.ParsingConfig, sql).ToQueryString();
+        results = await db.Employees.Where(parser.ParsingConfig, sql).ToListAsync();
         Assert.Single(results);
     }
 
@@ -142,8 +149,8 @@ public async Task CanHandleEmptyTokens()
 
         string sql = await parser.ToDynamicLinqAsync("test", context);
         _logger.LogInformation(sql);
-        string sqlActual = db.Employees.Where(sql).ToQueryString();
-        var results = await db.Employees.Where(sql).ToListAsync();
+        string sqlActual = db.Employees.Where(parser.ParsingConfig, sql).ToQueryString();
+        var results = await db.Employees.Where(parser.ParsingConfig, sql).ToListAsync();
         Assert.Empty(results);
     }
 
@@ -302,11 +309,11 @@ public async Task CanUseCollectionDefaultFields()
 
         var context = parser.GetContext(db.Employees.EntityType);
 
-        string sqlExpected = db.Employees.Where(e => e.Companies.Any(c => c.Name.StartsWith("acme"))).ToQueryString();
-        string sqlActual = db.Employees.Where("""Companies.Any(Name.StartsWith("acme"))""").ToQueryString();
+        string sqlExpected = db.Employees.Where(e => e.Companies.Any(c => EF.Functions.Contains(c.Name, "\"acme*\""))).ToQueryString();
+        string sqlActual = db.Employees.Where(parser.ParsingConfig, """Companies.Any(FTS.Contains(Name, "\"acme*\""))""").ToQueryString();
         Assert.Equal(sqlExpected, sqlActual);
         string sql = await parser.ToDynamicLinqAsync("acme", context);
-        sqlActual = db.Employees.Where(sql).ToQueryString();
+        sqlActual = db.Employees.Where(parser.ParsingConfig, sql).ToQueryString();
         Assert.Equal(sqlExpected, sqlActual);
     }
 
@@ -450,7 +457,7 @@ public IServiceProvider GetServiceProvider()
         parser.Configuration.UseEntityTypePropertyFilter(p => p.Name != nameof(Company.Description));
         parser.Configuration.AddQueryVisitor(new DynamicFieldVisitor());
         parser.Configuration.SetDefaultFields(["FullName"], SqlSearchOperator.Contains);
-        parser.Configuration.UseFullTextSearch();
+        parser.Configuration.SetFullTextFields(["Name", "FullName", "Title", "NationalPhoneNumber"]);
         services.AddSingleton(parser);
         return services.BuildServiceProvider();
     }
@@ -504,24 +511,23 @@ await db.Database.ExecuteSqlRawAsync(
             @"IF FULLTEXTSERVICEPROPERTY('IsFullTextInstalled') != 1
               BEGIN
                   RAISERROR('Full-Text Search is not installed', 16, 1);
-              END");
+              END
 
-        await db.Database.ExecuteSqlRawAsync(
-            @"IF NOT EXISTS (SELECT * FROM sys.fulltext_catalogs WHERE name = 'ftCatalog')
+              IF NOT EXISTS (SELECT * FROM sys.fulltext_catalogs WHERE name = 'ftCatalog')
               BEGIN
                   CREATE FULLTEXT CATALOG ftCatalog AS DEFAULT;
-              END");
+              END
 
-        await db.Database.ExecuteSqlRawAsync(
-            @"IF EXISTS (SELECT * FROM sys.fulltext_indexes WHERE object_id = OBJECT_ID('Employees'))
+              IF EXISTS (SELECT * FROM sys.fulltext_indexes WHERE object_id = OBJECT_ID('Employees'))
               BEGIN
                   DROP FULLTEXT INDEX ON Employees;
-              END");
+              END
 
-        await db.Database.ExecuteSqlRawAsync(
-            @"CREATE FULLTEXT INDEX ON Employees
+              CREATE FULLTEXT INDEX ON Employees
               (
-                  FullName LANGUAGE 1033
+                  FullName LANGUAGE 1033,
+                  NationalPhoneNumber LANGUAGE 1033,
+                  Title LANGUAGE 1033
               )
               KEY INDEX PK_Employees
               ON ftCatalog
@@ -530,6 +536,27 @@ ON ftCatalog
         return db;
     }
 
+    private async Task WaitForFullTextIndexAsync(DbContext db, string catalogName, int timeoutSeconds = 30)
+    {
+        var end = DateTime.UtcNow.AddSeconds(timeoutSeconds);
+
+        while (DateTime.UtcNow < end)
+        {
+            string sql = "SELECT FULLTEXTCATALOGPROPERTY(@catalogName, 'PopulateStatus') AS Value";
+
+            int status = await db.Database
+                .SqlQueryRaw<int>(sql, new SqlParameter("@catalogName", catalogName))
+                .SingleAsync();
+
+            if (status == 0)
+                return;
+
+            await Task.Delay(500);
+        }
+
+        throw new TimeoutException($"Full-text catalog '{catalogName}' didn't finish populating in time.");
+    }
+
     private async Task ParseAndValidateQuery(string query, string expected, bool isValid)
     {
 #if ENABLE_TRACING

Original file line number	Diff line number	Diff line change
`@@ -225,7 +225,7 @@ private void SetupQueryVisitorContextDefaults(IQueryVisitorContext context)`
`225`	`225`	`sqlContext.DateTimeParser = Configuration.DateTimeParser;`
`226`	`226`	`sqlContext.DateOnlyParser = Configuration.DateOnlyParser;`
`227`	`227`	`sqlContext.DefaultSearchOperator = Configuration.DefaultFieldsSearchOperator;`
`228`		`- sqlContext.FullTextSearchEnabled = Configuration.FullTextSearchEnabled;`
	`228`	`+ sqlContext.FullTextFields = Configuration.FullTextFields;`
`229`	`229`	`}`
`230`	`230`	`}`
`231`	`231`	`}`
Original file line number	Diff line number	Diff line change
`@@ -24,8 +24,7 @@ public SqlQueryParserConfiguration()`
`24`	`24`	`public ILoggerFactory LoggerFactory { get; private set; } = NullLoggerFactory.Instance;`
`25`	`25`	`public string[] DefaultFields { get; private set; }`
`26`	`26`	`public SqlSearchOperator DefaultFieldsSearchOperator { get; private set; } = SqlSearchOperator.StartsWith;`
`27`		`- public bool FullTextSearchEnabled { get; private set; } = false;`
`28`		`-`
	`27`	`+ public string[] FullTextFields { get; private set; }`
`29`	`28`	`public int MaxFieldDepth { get; private set; } = 10;`
`30`	`29`	`public QueryFieldResolver FieldResolver { get; private set; }`
`31`	`30`	`public Action<SearchTerm> SearchTokenizer { get; set; } = static _ => { };`
`@@ -61,9 +60,9 @@ public SqlQueryParserConfiguration SetSearchTokenizer(Action<SearchTerm> tokeniz`
`61`	`60`	`return this;`
`62`	`61`	`}`
`63`	`62`
`64`		`- public SqlQueryParserConfiguration UseFullTextSearch(bool useFullTextSearch = true)`
	`63`	`+ public SqlQueryParserConfiguration SetFullTextFields(string[] fields)`
`65`	`64`	`{`
`66`		`- FullTextSearchEnabled = useFullTextSearch;`
	`65`	`+ FullTextFields = fields;`
`67`	`66`	`return this;`
`68`	`67`	`}`
`69`	`68`
Original file line number	Diff line number	Diff line change
`@@ -8,7 +8,7 @@ public interface ISqlQueryVisitorContext : IQueryVisitorContext`
`8`	`8`	`{`
`9`	`9`	`List<EntityFieldInfo> Fields { get; set; }`
`10`	`10`	`SqlSearchOperator DefaultSearchOperator { get; set; }`
`11`		`- bool FullTextSearchEnabled { get; set; }`
	`11`	`+ string[] FullTextFields { get; set; }`
`12`	`12`	`Action<SearchTerm> SearchTokenizer { get; set; }`
`13`	`13`	`Func<string, string> DateTimeParser { get; set; }`
`14`	`14`	`Func<string, string> DateOnlyParser { get; set; }`
Original file line number	Diff line number	Diff line change
`@@ -11,7 +11,7 @@ public class SqlQueryVisitorContext : QueryVisitorContext, ISqlQueryVisitorConte`
`11`	`11`	`{`
`12`	`12`	`public List<EntityFieldInfo> Fields { get; set; }`
`13`	`13`	`public SqlSearchOperator DefaultSearchOperator { get; set; } = SqlSearchOperator.StartsWith;`
`14`		`- public bool FullTextSearchEnabled { get; set; } = false;`
	`14`	`+ public string[] FullTextFields { get; set; } = [];`
`15`	`15`	`public Action<SearchTerm> SearchTokenizer { get; set; } = static _ => { };`
`16`	`16`	`public Func<string, string> DateTimeParser { get; set; }`
`17`	`17`	`public Func<string, string> DateOnlyParser { get; set; }`