From 0ffafc0dae4d0ca2ac86d5343d2759dc1cdf4ae2 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Tue, 11 Nov 2025 20:28:53 -0600 Subject: [PATCH 1/8] Upgrade to EF 10 --- .../Foundatio.Parsers.SqlQueries.csproj | 9 +++------ tests/Directory.Build.props | 2 +- .../Foundatio.Parsers.SqlQueries.Tests.csproj | 6 ++++-- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/src/Foundatio.Parsers.SqlQueries/Foundatio.Parsers.SqlQueries.csproj b/src/Foundatio.Parsers.SqlQueries/Foundatio.Parsers.SqlQueries.csproj index cc7ed75..2fd0b59 100644 --- a/src/Foundatio.Parsers.SqlQueries/Foundatio.Parsers.SqlQueries.csproj +++ b/src/Foundatio.Parsers.SqlQueries/Foundatio.Parsers.SqlQueries.csproj @@ -1,15 +1,12 @@ - net8.0; + net10.0; - + - - - - + diff --git a/tests/Directory.Build.props b/tests/Directory.Build.props index 5fdff56..96d1b1c 100644 --- a/tests/Directory.Build.props +++ b/tests/Directory.Build.props @@ -6,7 +6,7 @@ $(NoWarn);CS1591;NU1701 - + diff --git a/tests/Foundatio.Parsers.SqlQueries.Tests/Foundatio.Parsers.SqlQueries.Tests.csproj b/tests/Foundatio.Parsers.SqlQueries.Tests/Foundatio.Parsers.SqlQueries.Tests.csproj index 3c49ac5..de035ca 100644 --- a/tests/Foundatio.Parsers.SqlQueries.Tests/Foundatio.Parsers.SqlQueries.Tests.csproj +++ b/tests/Foundatio.Parsers.SqlQueries.Tests/Foundatio.Parsers.SqlQueries.Tests.csproj @@ -1,5 +1,6 @@ + net10.0; $(NoWarn);CS8002; @@ -9,8 +10,9 @@ - - + + + From dff067795d082d4c7b751e8497d93c9fba830d1f Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Tue, 18 Nov 2025 19:13:28 -0600 Subject: [PATCH 2/8] Adding support for full text search --- Dockerfile | 23 +++++++++ docker-compose.yml | 4 +- .../Extensions/SqlNodeExtensions.cs | 44 ++++++++++++---- .../Foundatio.Parsers.SqlQueries.csproj | 2 +- .../SqlQueryParser.cs | 19 +++++++ .../SqlQueryParserConfiguration.cs | 11 +++- .../Visitors/ISqlQueryVisitorContext.cs | 2 + .../Visitors/SqlQueryVisitorContext.cs | 2 + .../SampleContext.cs | 11 ++++ .../SqlQueryParserTests.cs | 50 ++++++++++++++++++- 10 files changed, 154 insertions(+), 14 deletions(-) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..47514c9 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,23 @@ +FROM mcr.microsoft.com/mssql/server:2022-latest + +ARG SSID_PID=Developer + +ENV ACCEPT_EULA=Y +ENV SSID_PID=${SSID_PID} +ENV DEBIAN_FRONTEND=noninteractive +ENV DEBCONF_NONINTERACTIVE_SEEN=true + +USER root + +RUN apt-get update && \ + apt-get upgrade -y && \ + apt-get install -yq gnupg gnupg2 gnupg1 curl apt-transport-https && \ + curl https://packages.microsoft.com/keys/microsoft.asc -o /var/opt/mssql/ms-key.cer && \ + gpg --dearmor -o /etc/apt/trusted.gpg.d/microsoft.gpg /var/opt/mssql/ms-key.cer && \ + curl https://packages.microsoft.com/config/ubuntu/22.04/mssql-server-2022.list -o /etc/apt/sources.list.d/mssql-server-2022.list && \ + apt-get update && \ + apt-get install -y mssql-server-fts && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists + +ENTRYPOINT [ "/opt/mssql/bin/sqlservr" ] diff --git a/docker-compose.yml b/docker-compose.yml index a554fc2..a89cca8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -30,7 +30,9 @@ services: test: curl --write-out 'HTTP %{http_code}' --fail --silent --output /dev/null http://localhost:5601/api/status sqlserver: - image: mcr.microsoft.com/mssql/server:2025-latest + build: + context: . + dockerfile: Dockerfile ports: - "1433:1433" # login with sa:P@ssword1 environment: diff --git a/src/Foundatio.Parsers.SqlQueries/Extensions/SqlNodeExtensions.cs b/src/Foundatio.Parsers.SqlQueries/Extensions/SqlNodeExtensions.cs index bd894d5..71d8c33 100644 --- a/src/Foundatio.Parsers.SqlQueries/Extensions/SqlNodeExtensions.cs +++ b/src/Foundatio.Parsers.SqlQueries/Extensions/SqlNodeExtensions.cs @@ -142,7 +142,7 @@ public static string ToDynamicLinqString(this TermNode node, ISqlQueryVisitorCon { FieldInfo = fieldInfo, Term = node.Term, - Operator = SqlSearchOperator.StartsWith + Operator = context.DefaultSearchOperator }; fieldTerms[fieldInfo] = searchTerm; } @@ -186,10 +186,23 @@ public static string ToDynamicLinqString(this TermNode node, ISqlQueryVisitorCon { builder.Append(i.IsFirst ? "(" : " OR "); builder.Append(fieldPrefix); - builder.Append(kvp.Key.Name); - builder.Append(".Contains("); - AppendField(builder, kvp.Key, token, context); - builder.Append(")"); + + if (context.FullTextSearchEnabled) + { + builder.Append("FTS.Contains("); + builder.Append(kvp.Key.Name); + builder.Append(", "); + AppendField(builder, kvp.Key, token, context); + builder.Append(")"); + } + else + { + builder.Append(kvp.Key.Name); + builder.Append(".Contains("); + AppendField(builder, kvp.Key, token, context); + builder.Append(")"); + } + builder.Append(fieldSuffix); if (i.IsLast) builder.Append(")"); @@ -201,10 +214,23 @@ public static string ToDynamicLinqString(this TermNode node, ISqlQueryVisitorCon { builder.Append(i.IsFirst ? "(" : " OR "); builder.Append(fieldPrefix); - builder.Append(kvp.Key.Name); - builder.Append(".StartsWith("); - AppendField(builder, kvp.Key, token, context); - builder.Append(")"); + + if (context.FullTextSearchEnabled) + { + builder.Append("FTS.Contains("); + builder.Append(kvp.Key.Name); + builder.Append(", "); + AppendField(builder, kvp.Key, token + "*", context); + builder.Append(")"); + } + else + { + builder.Append(kvp.Key.Name); + builder.Append(".StartsWith("); + AppendField(builder, kvp.Key, token, context); + builder.Append(")"); + } + builder.Append(fieldSuffix); if (i.IsLast) builder.Append(")"); diff --git a/src/Foundatio.Parsers.SqlQueries/Foundatio.Parsers.SqlQueries.csproj b/src/Foundatio.Parsers.SqlQueries/Foundatio.Parsers.SqlQueries.csproj index 2fd0b59..c47323a 100644 --- a/src/Foundatio.Parsers.SqlQueries/Foundatio.Parsers.SqlQueries.csproj +++ b/src/Foundatio.Parsers.SqlQueries/Foundatio.Parsers.SqlQueries.csproj @@ -4,7 +4,7 @@ net10.0; - + diff --git a/src/Foundatio.Parsers.SqlQueries/SqlQueryParser.cs b/src/Foundatio.Parsers.SqlQueries/SqlQueryParser.cs index 14bab6d..790d8da 100644 --- a/src/Foundatio.Parsers.SqlQueries/SqlQueryParser.cs +++ b/src/Foundatio.Parsers.SqlQueries/SqlQueryParser.cs @@ -3,6 +3,8 @@ using System.Collections.Generic; using System.ComponentModel.DataAnnotations; using System.Linq; +using System.Linq.Dynamic.Core; +using System.Linq.Dynamic.Core.CustomTypeProviders; using System.Threading.Tasks; using Foundatio.Parsers.LuceneQueries; using Foundatio.Parsers.LuceneQueries.Extensions; @@ -10,6 +12,7 @@ using Foundatio.Parsers.LuceneQueries.Visitors; using Foundatio.Parsers.SqlQueries.Extensions; using Foundatio.Parsers.SqlQueries.Visitors; +using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Metadata; using Pegasus.Common; @@ -25,6 +28,10 @@ public SqlQueryParser(Action configure = null) } public SqlQueryParserConfiguration Configuration { get; } + public ParsingConfig ParsingConfig { get; } = new() + { + CustomTypeProvider = new DynamicLinqTypeProvider() + }; public override async Task ParseAsync(string query, IQueryVisitorContext context = null) { @@ -217,6 +224,18 @@ private void SetupQueryVisitorContextDefaults(IQueryVisitorContext context) sqlContext.SearchTokenizer = Configuration.SearchTokenizer; sqlContext.DateTimeParser = Configuration.DateTimeParser; sqlContext.DateOnlyParser = Configuration.DateOnlyParser; + sqlContext.DefaultSearchOperator = Configuration.DefaultFieldsSearchOperator; + sqlContext.FullTextSearchEnabled = Configuration.FullTextSearchEnabled; } } } + +public static class FTS +{ + public static bool Contains(string propertyValue, string searchTerm) + { + return EF.Functions.Contains(propertyValue, searchTerm); + } +} + +public class DynamicLinqTypeProvider() : DefaultDynamicLinqCustomTypeProvider(ParsingConfig.Default, [ typeof(EF), typeof(FTS) ]); diff --git a/src/Foundatio.Parsers.SqlQueries/SqlQueryParserConfiguration.cs b/src/Foundatio.Parsers.SqlQueries/SqlQueryParserConfiguration.cs index bc46037..e89b29e 100644 --- a/src/Foundatio.Parsers.SqlQueries/SqlQueryParserConfiguration.cs +++ b/src/Foundatio.Parsers.SqlQueries/SqlQueryParserConfiguration.cs @@ -23,6 +23,8 @@ public SqlQueryParserConfiguration() public ILoggerFactory LoggerFactory { get; private set; } = NullLoggerFactory.Instance; public string[] DefaultFields { get; private set; } + public SqlSearchOperator DefaultFieldsSearchOperator { get; private set; } = SqlSearchOperator.StartsWith; + public bool FullTextSearchEnabled { get; private set; } = false; public int MaxFieldDepth { get; private set; } = 10; public QueryFieldResolver FieldResolver { get; private set; } @@ -46,9 +48,10 @@ public SqlQueryParserConfiguration SetLoggerFactory(ILoggerFactory loggerFactory return this; } - public SqlQueryParserConfiguration SetDefaultFields(string[] fields) + public SqlQueryParserConfiguration SetDefaultFields(string[] fields, SqlSearchOperator op = SqlSearchOperator.StartsWith) { DefaultFields = fields; + DefaultFieldsSearchOperator = op; return this; } @@ -58,6 +61,12 @@ public SqlQueryParserConfiguration SetSearchTokenizer(Action tokeniz return this; } + public SqlQueryParserConfiguration UseFullTextSearch(bool useFullTextSearch = true) + { + FullTextSearchEnabled = useFullTextSearch; + return this; + } + public SqlQueryParserConfiguration SetDateTimeParser(Func dateTimeParser) { DateTimeParser = dateTimeParser; diff --git a/src/Foundatio.Parsers.SqlQueries/Visitors/ISqlQueryVisitorContext.cs b/src/Foundatio.Parsers.SqlQueries/Visitors/ISqlQueryVisitorContext.cs index 6bd8e7d..fe436ee 100644 --- a/src/Foundatio.Parsers.SqlQueries/Visitors/ISqlQueryVisitorContext.cs +++ b/src/Foundatio.Parsers.SqlQueries/Visitors/ISqlQueryVisitorContext.cs @@ -7,6 +7,8 @@ namespace Foundatio.Parsers.SqlQueries.Visitors; public interface ISqlQueryVisitorContext : IQueryVisitorContext { List Fields { get; set; } + SqlSearchOperator DefaultSearchOperator { get; set; } + bool FullTextSearchEnabled { get; set; } Action SearchTokenizer { get; set; } Func DateTimeParser { get; set; } Func DateOnlyParser { get; set; } diff --git a/src/Foundatio.Parsers.SqlQueries/Visitors/SqlQueryVisitorContext.cs b/src/Foundatio.Parsers.SqlQueries/Visitors/SqlQueryVisitorContext.cs index fb82f2f..03e7bab 100644 --- a/src/Foundatio.Parsers.SqlQueries/Visitors/SqlQueryVisitorContext.cs +++ b/src/Foundatio.Parsers.SqlQueries/Visitors/SqlQueryVisitorContext.cs @@ -10,6 +10,8 @@ namespace Foundatio.Parsers.SqlQueries.Visitors; public class SqlQueryVisitorContext : QueryVisitorContext, ISqlQueryVisitorContext { public List Fields { get; set; } + public SqlSearchOperator DefaultSearchOperator { get; set; } = SqlSearchOperator.StartsWith; + public bool FullTextSearchEnabled { get; set; } = false; public Action SearchTokenizer { get; set; } = static _ => { }; public Func DateTimeParser { get; set; } public Func DateOnlyParser { get; set; } diff --git a/tests/Foundatio.Parsers.SqlQueries.Tests/SampleContext.cs b/tests/Foundatio.Parsers.SqlQueries.Tests/SampleContext.cs index e99d5cc..b8bc04a 100644 --- a/tests/Foundatio.Parsers.SqlQueries.Tests/SampleContext.cs +++ b/tests/Foundatio.Parsers.SqlQueries.Tests/SampleContext.cs @@ -1,6 +1,8 @@ using System; using System.Collections.Generic; +using System.Linq; using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Query.SqlExpressions; namespace Foundatio.Parsers.SqlQueries.Tests; @@ -34,6 +36,15 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) modelBuilder.Entity().Property(e => e.BooleanValue).IsSparse(); modelBuilder.Entity().Property(e => e.NumberValue).HasColumnType("decimal").HasPrecision(15, 3).IsSparse(); modelBuilder.Entity().HasIndex(e => new { e.StringValue, e.DateValue, e.MoneyValue, e.BooleanValue, e.NumberValue }); + + modelBuilder.HasDbFunction(typeof(FTS).GetMethod(nameof(FTS.Contains))!) + .HasTranslation(args => new SqlFunctionExpression( + "CONTAINS", + args, + true, + args.Select(a => false).ToList(), + typeof(bool), + null)); } } diff --git a/tests/Foundatio.Parsers.SqlQueries.Tests/SqlQueryParserTests.cs b/tests/Foundatio.Parsers.SqlQueries.Tests/SqlQueryParserTests.cs index 2bb6ea1..109cac4 100644 --- a/tests/Foundatio.Parsers.SqlQueries.Tests/SqlQueryParserTests.cs +++ b/tests/Foundatio.Parsers.SqlQueries.Tests/SqlQueryParserTests.cs @@ -377,6 +377,22 @@ public async Task CanUseSkipNavigationFields() Assert.Single(companies); } + [Fact] + public async Task CanUseLike() + { + var sp = GetServiceProvider(); + await using var db = await GetSampleContextWithDataAsync(sp); + var parser = sp.GetRequiredService(); + + var context = parser.GetContext(db.Employees.EntityType); + string sqlExpected = db.Employees.Where(e => EF.Functions.Contains(e.FullName, "john")).ToQueryString(); + string sqlActual = db.Employees.Where(parser.ParsingConfig, """FTS.Contains(FullName, "john")""").ToQueryString(); + Assert.Equal(sqlExpected, sqlActual); + string sql = await parser.ToDynamicLinqAsync("john", context); + sqlActual = db.Employees.Where(parser.ParsingConfig, sql).ToQueryString(); + Assert.Equal(sqlExpected, sqlActual); + } + [Fact] public async Task CanGenerateSql() { @@ -384,6 +400,7 @@ public async Task CanGenerateSql() await using var db = await GetSampleContextWithDataAsync(sp); var parser = sp.GetRequiredService(); + var efFunctions = EF.Functions; var context = parser.GetContext(db.Employees.EntityType); context.Fields.Add(new EntityFieldInfo { Name = "age", FullName = "age", IsNumber = true, Data = { { "DataDefinitionId", 1 } } }); context.ValidationOptions.AllowedFields.Add("age"); @@ -391,8 +408,8 @@ public async Task CanGenerateSql() string sqlExpected = db.Employees.Where(e => e.Companies.Any(c => c.Name == "acme") && e.DataValues.Any(dv => dv.DataDefinitionId == 1 && dv.NumberValue == 30)).ToQueryString(); string sqlActual = db.Employees.Where("""Companies.Any(Name = "acme") AND DataValues.Any(DataDefinitionId = 1 AND NumberValue = 30) """).ToQueryString(); Assert.Equal(sqlExpected, sqlActual); - string sql = await parser.ToDynamicLinqAsync("companies.name:acme age:30", context); - sqlActual = db.Employees.Where(sql).ToQueryString(); + string sql = await parser.ToDynamicLinqAsync("EF.Contains(@0, companies.name, @1)companies.name:acme age:30", context); + sqlActual = db.Employees.Where(sql, efFunctions).ToQueryString(); Assert.Equal(sqlExpected, sqlActual); var q = db.Employees.AsNoTracking(); @@ -433,6 +450,8 @@ public IServiceProvider GetServiceProvider() var parser = new SqlQueryParser(); parser.Configuration.UseEntityTypePropertyFilter(p => p.Name != nameof(Company.Description)); parser.Configuration.AddQueryVisitor(new DynamicFieldVisitor()); + parser.Configuration.SetDefaultFields(["FullName"], SqlSearchOperator.Contains); + parser.Configuration.UseFullTextSearch(); services.AddSingleton(parser); return services.BuildServiceProvider(); } @@ -482,6 +501,33 @@ public async Task GetSampleContextWithDataAsync(IServiceProvider }); await db.SaveChangesAsync(); + var result = await db.Database.ExecuteSqlRawAsync( + @"IF FULLTEXTSERVICEPROPERTY('IsFullTextInstalled') != 1 + BEGIN + RAISERROR('Full-Text Search is not installed', 16, 1); + END"); + + result = await db.Database.ExecuteSqlRawAsync( + @"IF NOT EXISTS (SELECT * FROM sys.fulltext_catalogs WHERE name = 'ftCatalog') + BEGIN + CREATE FULLTEXT CATALOG ftCatalog AS DEFAULT; + END"); + + result = await db.Database.ExecuteSqlRawAsync( + @"IF EXISTS (SELECT * FROM sys.fulltext_indexes WHERE object_id = OBJECT_ID('Employees')) + BEGIN + DROP FULLTEXT INDEX ON Employees; + END"); + + result = await db.Database.ExecuteSqlRawAsync( + @"CREATE FULLTEXT INDEX ON Employees + ( + FullName LANGUAGE 1033 + ) + KEY INDEX PK_Employees + ON ftCatalog + WITH (CHANGE_TRACKING = AUTO, STOPLIST = SYSTEM);"); + return db; } From 73310a17bb76c4994c42ac0c4a8ef4efe97348e7 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Tue, 18 Nov 2025 19:43:17 -0600 Subject: [PATCH 3/8] Revert --- .../SqlQueryParserTests.cs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/Foundatio.Parsers.SqlQueries.Tests/SqlQueryParserTests.cs b/tests/Foundatio.Parsers.SqlQueries.Tests/SqlQueryParserTests.cs index 109cac4..b2bbaa1 100644 --- a/tests/Foundatio.Parsers.SqlQueries.Tests/SqlQueryParserTests.cs +++ b/tests/Foundatio.Parsers.SqlQueries.Tests/SqlQueryParserTests.cs @@ -400,7 +400,6 @@ public async Task CanGenerateSql() await using var db = await GetSampleContextWithDataAsync(sp); var parser = sp.GetRequiredService(); - var efFunctions = EF.Functions; var context = parser.GetContext(db.Employees.EntityType); context.Fields.Add(new EntityFieldInfo { Name = "age", FullName = "age", IsNumber = true, Data = { { "DataDefinitionId", 1 } } }); context.ValidationOptions.AllowedFields.Add("age"); @@ -408,8 +407,8 @@ public async Task CanGenerateSql() string sqlExpected = db.Employees.Where(e => e.Companies.Any(c => c.Name == "acme") && e.DataValues.Any(dv => dv.DataDefinitionId == 1 && dv.NumberValue == 30)).ToQueryString(); string sqlActual = db.Employees.Where("""Companies.Any(Name = "acme") AND DataValues.Any(DataDefinitionId = 1 AND NumberValue = 30) """).ToQueryString(); Assert.Equal(sqlExpected, sqlActual); - string sql = await parser.ToDynamicLinqAsync("EF.Contains(@0, companies.name, @1)companies.name:acme age:30", context); - sqlActual = db.Employees.Where(sql, efFunctions).ToQueryString(); + string sql = await parser.ToDynamicLinqAsync("companies.name:acme age:30", context); + sqlActual = db.Employees.Where(sql).ToQueryString(); Assert.Equal(sqlExpected, sqlActual); var q = db.Employees.AsNoTracking(); From fd9302fc1bfda06c22c7b3d06ccbc2ee8ffcf4df Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Tue, 18 Nov 2025 20:47:02 -0600 Subject: [PATCH 4/8] Potential fix for pull request finding 'Useless assignment to local variable' Co-authored-by: Copilot Autofix powered by AI <223894421+github-code-quality[bot]@users.noreply.github.com> --- .../SqlQueryParserTests.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/Foundatio.Parsers.SqlQueries.Tests/SqlQueryParserTests.cs b/tests/Foundatio.Parsers.SqlQueries.Tests/SqlQueryParserTests.cs index b2bbaa1..22510fe 100644 --- a/tests/Foundatio.Parsers.SqlQueries.Tests/SqlQueryParserTests.cs +++ b/tests/Foundatio.Parsers.SqlQueries.Tests/SqlQueryParserTests.cs @@ -500,25 +500,25 @@ public async Task GetSampleContextWithDataAsync(IServiceProvider }); await db.SaveChangesAsync(); - var result = await db.Database.ExecuteSqlRawAsync( + await db.Database.ExecuteSqlRawAsync( @"IF FULLTEXTSERVICEPROPERTY('IsFullTextInstalled') != 1 BEGIN RAISERROR('Full-Text Search is not installed', 16, 1); END"); - result = await db.Database.ExecuteSqlRawAsync( + await db.Database.ExecuteSqlRawAsync( @"IF NOT EXISTS (SELECT * FROM sys.fulltext_catalogs WHERE name = 'ftCatalog') BEGIN CREATE FULLTEXT CATALOG ftCatalog AS DEFAULT; END"); - result = await db.Database.ExecuteSqlRawAsync( + await db.Database.ExecuteSqlRawAsync( @"IF EXISTS (SELECT * FROM sys.fulltext_indexes WHERE object_id = OBJECT_ID('Employees')) BEGIN DROP FULLTEXT INDEX ON Employees; END"); - result = await db.Database.ExecuteSqlRawAsync( + await db.Database.ExecuteSqlRawAsync( @"CREATE FULLTEXT INDEX ON Employees ( FullName LANGUAGE 1033 From 4b2d89a0c6ed8708134bf0b7d024c81e632df2a1 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Tue, 18 Nov 2025 23:06:56 -0600 Subject: [PATCH 5/8] Ability to control which fields are full text indexed --- .../Extensions/SqlNodeExtensions.cs | 48 +++++++--- .../SqlQueryParser.cs | 2 +- .../SqlQueryParserConfiguration.cs | 7 +- .../Visitors/ISqlQueryVisitorContext.cs | 2 +- .../Visitors/SqlQueryVisitorContext.cs | 2 +- .../SqlQueryParserTests.cs | 87 ++++++++++++------- 6 files changed, 100 insertions(+), 48 deletions(-) diff --git a/src/Foundatio.Parsers.SqlQueries/Extensions/SqlNodeExtensions.cs b/src/Foundatio.Parsers.SqlQueries/Extensions/SqlNodeExtensions.cs index 71d8c33..70f5c2c 100644 --- a/src/Foundatio.Parsers.SqlQueries/Extensions/SqlNodeExtensions.cs +++ b/src/Foundatio.Parsers.SqlQueries/Extensions/SqlNodeExtensions.cs @@ -187,7 +187,7 @@ public static string ToDynamicLinqString(this TermNode node, ISqlQueryVisitorCon builder.Append(i.IsFirst ? "(" : " OR "); builder.Append(fieldPrefix); - if (context.FullTextSearchEnabled) + if (context.FullTextFields.Contains(kvp.Key.Name, StringComparer.OrdinalIgnoreCase)) { builder.Append("FTS.Contains("); builder.Append(kvp.Key.Name); @@ -215,12 +215,12 @@ public static string ToDynamicLinqString(this TermNode node, ISqlQueryVisitorCon builder.Append(i.IsFirst ? "(" : " OR "); builder.Append(fieldPrefix); - if (context.FullTextSearchEnabled) + if (context.FullTextFields.Contains(kvp.Key.Name, StringComparer.OrdinalIgnoreCase)) { builder.Append("FTS.Contains("); builder.Append(kvp.Key.Name); builder.Append(", "); - AppendField(builder, kvp.Key, token + "*", context); + AppendField(builder, kvp.Key, "\\\"" + token + "*\\\"", context); builder.Append(")"); } else @@ -266,19 +266,45 @@ public static string ToDynamicLinqString(this TermNode node, ISqlQueryVisitorCon else if (searchOperator == SqlSearchOperator.Contains) { builder.Append(fieldPrefix); - builder.Append(field.Name); - builder.Append(".Contains("); - AppendField(builder, field, node.Term, context); - builder.Append(")"); + + if (context.FullTextFields.Contains(field.Name, StringComparer.OrdinalIgnoreCase)) + { + builder.Append("FTS.Contains("); + builder.Append(field.Name); + builder.Append(", "); + AppendField(builder, field, node.Term, context); + builder.Append(")"); + } + else + { + builder.Append(field.Name); + builder.Append(".Contains("); + AppendField(builder, field, node.Term, context); + builder.Append(")"); + } + builder.Append(fieldSuffix); } else { builder.Append(fieldPrefix); - builder.Append(field.Name); - builder.Append(".StartsWith("); - AppendField(builder, field, node.Term, context); - builder.Append(")"); + + if (context.FullTextFields.Contains(field.Name, StringComparer.OrdinalIgnoreCase)) + { + builder.Append("FTS.Contains("); + builder.Append(field.Name); + builder.Append(", "); + AppendField(builder, field, "\\\"" + node.Term + "*\\\"", context); + builder.Append(")"); + } + else + { + builder.Append(field.Name); + builder.Append(".Contains("); + AppendField(builder, field, node.Term, context); + builder.Append(")"); + } + builder.Append(fieldSuffix); } diff --git a/src/Foundatio.Parsers.SqlQueries/SqlQueryParser.cs b/src/Foundatio.Parsers.SqlQueries/SqlQueryParser.cs index 790d8da..4576b40 100644 --- a/src/Foundatio.Parsers.SqlQueries/SqlQueryParser.cs +++ b/src/Foundatio.Parsers.SqlQueries/SqlQueryParser.cs @@ -225,7 +225,7 @@ private void SetupQueryVisitorContextDefaults(IQueryVisitorContext context) sqlContext.DateTimeParser = Configuration.DateTimeParser; sqlContext.DateOnlyParser = Configuration.DateOnlyParser; sqlContext.DefaultSearchOperator = Configuration.DefaultFieldsSearchOperator; - sqlContext.FullTextSearchEnabled = Configuration.FullTextSearchEnabled; + sqlContext.FullTextFields = Configuration.FullTextFields; } } } diff --git a/src/Foundatio.Parsers.SqlQueries/SqlQueryParserConfiguration.cs b/src/Foundatio.Parsers.SqlQueries/SqlQueryParserConfiguration.cs index e89b29e..8df5ff2 100644 --- a/src/Foundatio.Parsers.SqlQueries/SqlQueryParserConfiguration.cs +++ b/src/Foundatio.Parsers.SqlQueries/SqlQueryParserConfiguration.cs @@ -24,8 +24,7 @@ public SqlQueryParserConfiguration() public ILoggerFactory LoggerFactory { get; private set; } = NullLoggerFactory.Instance; public string[] DefaultFields { get; private set; } public SqlSearchOperator DefaultFieldsSearchOperator { get; private set; } = SqlSearchOperator.StartsWith; - public bool FullTextSearchEnabled { get; private set; } = false; - + public string[] FullTextFields { get; private set; } public int MaxFieldDepth { get; private set; } = 10; public QueryFieldResolver FieldResolver { get; private set; } public Action SearchTokenizer { get; set; } = static _ => { }; @@ -61,9 +60,9 @@ public SqlQueryParserConfiguration SetSearchTokenizer(Action tokeniz return this; } - public SqlQueryParserConfiguration UseFullTextSearch(bool useFullTextSearch = true) + public SqlQueryParserConfiguration SetFullTextFields(string[] fields) { - FullTextSearchEnabled = useFullTextSearch; + FullTextFields = fields; return this; } diff --git a/src/Foundatio.Parsers.SqlQueries/Visitors/ISqlQueryVisitorContext.cs b/src/Foundatio.Parsers.SqlQueries/Visitors/ISqlQueryVisitorContext.cs index fe436ee..807e20d 100644 --- a/src/Foundatio.Parsers.SqlQueries/Visitors/ISqlQueryVisitorContext.cs +++ b/src/Foundatio.Parsers.SqlQueries/Visitors/ISqlQueryVisitorContext.cs @@ -8,7 +8,7 @@ public interface ISqlQueryVisitorContext : IQueryVisitorContext { List Fields { get; set; } SqlSearchOperator DefaultSearchOperator { get; set; } - bool FullTextSearchEnabled { get; set; } + string[] FullTextFields { get; set; } Action SearchTokenizer { get; set; } Func DateTimeParser { get; set; } Func DateOnlyParser { get; set; } diff --git a/src/Foundatio.Parsers.SqlQueries/Visitors/SqlQueryVisitorContext.cs b/src/Foundatio.Parsers.SqlQueries/Visitors/SqlQueryVisitorContext.cs index 03e7bab..e1e73fe 100644 --- a/src/Foundatio.Parsers.SqlQueries/Visitors/SqlQueryVisitorContext.cs +++ b/src/Foundatio.Parsers.SqlQueries/Visitors/SqlQueryVisitorContext.cs @@ -11,7 +11,7 @@ public class SqlQueryVisitorContext : QueryVisitorContext, ISqlQueryVisitorConte { public List Fields { get; set; } public SqlSearchOperator DefaultSearchOperator { get; set; } = SqlSearchOperator.StartsWith; - public bool FullTextSearchEnabled { get; set; } = false; + public string[] FullTextFields { get; set; } = []; public Action SearchTokenizer { get; set; } = static _ => { }; public Func DateTimeParser { get; set; } public Func DateOnlyParser { get; set; } diff --git a/tests/Foundatio.Parsers.SqlQueries.Tests/SqlQueryParserTests.cs b/tests/Foundatio.Parsers.SqlQueries.Tests/SqlQueryParserTests.cs index 22510fe..9b056c7 100644 --- a/tests/Foundatio.Parsers.SqlQueries.Tests/SqlQueryParserTests.cs +++ b/tests/Foundatio.Parsers.SqlQueries.Tests/SqlQueryParserTests.cs @@ -7,6 +7,7 @@ using Foundatio.Parsers.LuceneQueries.Visitors; using Foundatio.Parsers.SqlQueries.Visitors; using Foundatio.Xunit; +using Microsoft.Data.SqlClient; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Infrastructure; using Microsoft.Extensions.DependencyInjection; @@ -70,14 +71,17 @@ public async Task CanSearchDefaultFields() var context = parser.GetContext(db.Employees.EntityType); - string sqlExpected = db.Employees.Where(e => e.FullName.StartsWith("John") || e.Title.StartsWith("John")).ToQueryString(); - string sqlActual = db.Employees.Where("""FullName.StartsWith("John") || Title.StartsWith("John") """).ToQueryString(); + string sqlExpected = db.Employees.Where(e => EF.Functions.Contains(e.FullName, "\"John*\"") || EF.Functions.Contains(e.Title, "\"John*\"")).ToQueryString(); + string sqlActual = db.Employees.Where(parser.ParsingConfig, """FTS.Contains(FullName, "\"John*\"") || FTS.Contains(Title, "\"John*\"") """).ToQueryString(); Assert.Equal(sqlExpected, sqlActual); string sql = await parser.ToDynamicLinqAsync("John", context); - sqlActual = db.Employees.Where(sql).ToQueryString(); - var results = await db.Employees.Where(sql).ToListAsync(); - Assert.Single(results); + sqlActual = db.Employees.Where(parser.ParsingConfig, sql).ToQueryString(); Assert.Equal(sqlExpected, sqlActual); + + await WaitForFullTextIndexAsync(db, "ftCatalog"); + + var results = await db.Employees.Where(parser.ParsingConfig, sql).ToListAsync(); + Assert.Single(results); } [Fact] @@ -101,28 +105,31 @@ public async Task CanSearchWithTokenizer() var context = parser.GetContext(db.Employees.EntityType); - string sqlExpected = db.Employees.Where(e => e.NationalPhoneNumber.StartsWith("2142222222")).ToQueryString(); - string sqlActual = db.Employees.Where("NationalPhoneNumber.StartsWith(\"2142222222\")").ToQueryString(); + string sqlExpected = db.Employees.Where(e => EF.Functions.Contains(e.NationalPhoneNumber, "\"2142222222*\"")).ToQueryString(); + string sqlActual = db.Employees.Where(parser.ParsingConfig, "FTS.Contains(NationalPhoneNumber, \"\\\"2142222222*\\\"\")").ToQueryString(); Assert.Equal(sqlExpected, sqlActual); string sql = await parser.ToDynamicLinqAsync("214-222-2222", context); _logger.LogInformation(sql); - sqlActual = db.Employees.Where(sql).ToQueryString(); - var results = await db.Employees.Where(sql).ToListAsync(); - Assert.Single(results); Assert.Equal(sqlExpected, sqlActual); + await WaitForFullTextIndexAsync(db, "ftCatalog"); + + sqlActual = db.Employees.Where(parser.ParsingConfig, sql).ToQueryString(); + var results = await db.Employees.Where(parser.ParsingConfig, sql).ToListAsync(); + Assert.Single(results); + sql = await parser.ToDynamicLinqAsync("2142222222", context); _logger.LogInformation(sql); - sqlActual = db.Employees.Where(sql).ToQueryString(); - results = await db.Employees.Where(sql).ToListAsync(); + sqlActual = db.Employees.Where(parser.ParsingConfig, sql).ToQueryString(); + results = await db.Employees.Where(parser.ParsingConfig, sql).ToListAsync(); Assert.Single(results); Assert.Equal(sqlExpected, sqlActual); sql = await parser.ToDynamicLinqAsync("21422", context); _logger.LogInformation(sql); - sqlActual = db.Employees.Where(sql).ToQueryString(); - results = await db.Employees.Where(sql).ToListAsync(); + sqlActual = db.Employees.Where(parser.ParsingConfig, sql).ToQueryString(); + results = await db.Employees.Where(parser.ParsingConfig, sql).ToListAsync(); Assert.Single(results); } @@ -142,8 +149,8 @@ public async Task CanHandleEmptyTokens() string sql = await parser.ToDynamicLinqAsync("test", context); _logger.LogInformation(sql); - string sqlActual = db.Employees.Where(sql).ToQueryString(); - var results = await db.Employees.Where(sql).ToListAsync(); + string sqlActual = db.Employees.Where(parser.ParsingConfig, sql).ToQueryString(); + var results = await db.Employees.Where(parser.ParsingConfig, sql).ToListAsync(); Assert.Empty(results); } @@ -302,11 +309,11 @@ public async Task CanUseCollectionDefaultFields() var context = parser.GetContext(db.Employees.EntityType); - string sqlExpected = db.Employees.Where(e => e.Companies.Any(c => c.Name.StartsWith("acme"))).ToQueryString(); - string sqlActual = db.Employees.Where("""Companies.Any(Name.StartsWith("acme"))""").ToQueryString(); + string sqlExpected = db.Employees.Where(e => e.Companies.Any(c => EF.Functions.Contains(c.Name, "\"acme*\""))).ToQueryString(); + string sqlActual = db.Employees.Where(parser.ParsingConfig, """Companies.Any(FTS.Contains(Name, "\"acme*\""))""").ToQueryString(); Assert.Equal(sqlExpected, sqlActual); string sql = await parser.ToDynamicLinqAsync("acme", context); - sqlActual = db.Employees.Where(sql).ToQueryString(); + sqlActual = db.Employees.Where(parser.ParsingConfig, sql).ToQueryString(); Assert.Equal(sqlExpected, sqlActual); } @@ -450,7 +457,7 @@ public IServiceProvider GetServiceProvider() parser.Configuration.UseEntityTypePropertyFilter(p => p.Name != nameof(Company.Description)); parser.Configuration.AddQueryVisitor(new DynamicFieldVisitor()); parser.Configuration.SetDefaultFields(["FullName"], SqlSearchOperator.Contains); - parser.Configuration.UseFullTextSearch(); + parser.Configuration.SetFullTextFields(["Name", "FullName", "Title", "NationalPhoneNumber"]); services.AddSingleton(parser); return services.BuildServiceProvider(); } @@ -504,24 +511,23 @@ await db.Database.ExecuteSqlRawAsync( @"IF FULLTEXTSERVICEPROPERTY('IsFullTextInstalled') != 1 BEGIN RAISERROR('Full-Text Search is not installed', 16, 1); - END"); + END - await db.Database.ExecuteSqlRawAsync( - @"IF NOT EXISTS (SELECT * FROM sys.fulltext_catalogs WHERE name = 'ftCatalog') + IF NOT EXISTS (SELECT * FROM sys.fulltext_catalogs WHERE name = 'ftCatalog') BEGIN CREATE FULLTEXT CATALOG ftCatalog AS DEFAULT; - END"); + END - await db.Database.ExecuteSqlRawAsync( - @"IF EXISTS (SELECT * FROM sys.fulltext_indexes WHERE object_id = OBJECT_ID('Employees')) + IF EXISTS (SELECT * FROM sys.fulltext_indexes WHERE object_id = OBJECT_ID('Employees')) BEGIN DROP FULLTEXT INDEX ON Employees; - END"); + END - await db.Database.ExecuteSqlRawAsync( - @"CREATE FULLTEXT INDEX ON Employees + CREATE FULLTEXT INDEX ON Employees ( - FullName LANGUAGE 1033 + FullName LANGUAGE 1033, + NationalPhoneNumber LANGUAGE 1033, + Title LANGUAGE 1033 ) KEY INDEX PK_Employees ON ftCatalog @@ -530,6 +536,27 @@ ON ftCatalog return db; } + private async Task WaitForFullTextIndexAsync(DbContext db, string catalogName, int timeoutSeconds = 30) + { + var end = DateTime.UtcNow.AddSeconds(timeoutSeconds); + + while (DateTime.UtcNow < end) + { + string sql = "SELECT FULLTEXTCATALOGPROPERTY(@catalogName, 'PopulateStatus') AS Value"; + + int status = await db.Database + .SqlQueryRaw(sql, new SqlParameter("@catalogName", catalogName)) + .SingleAsync(); + + if (status == 0) + return; + + await Task.Delay(500); + } + + throw new TimeoutException($"Full-text catalog '{catalogName}' didn't finish populating in time."); + } + private async Task ParseAndValidateQuery(string query, string expected, bool isValid) { #if ENABLE_TRACING From 35926d70d22f4aa1aeeb0ed403781c204f47c1de Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Wed, 19 Nov 2025 11:37:05 -0600 Subject: [PATCH 6/8] Wait for SQL --- .../SqlQueryParserTests.cs | 35 +++++++++++ .../SqlWaiter.cs | 63 +++++++++++++++++++ 2 files changed, 98 insertions(+) create mode 100644 tests/Foundatio.Parsers.SqlQueries.Tests/SqlWaiter.cs diff --git a/tests/Foundatio.Parsers.SqlQueries.Tests/SqlQueryParserTests.cs b/tests/Foundatio.Parsers.SqlQueries.Tests/SqlQueryParserTests.cs index 9b056c7..40bda30 100644 --- a/tests/Foundatio.Parsers.SqlQueries.Tests/SqlQueryParserTests.cs +++ b/tests/Foundatio.Parsers.SqlQueries.Tests/SqlQueryParserTests.cs @@ -2,6 +2,7 @@ using System.ComponentModel.DataAnnotations; using System.Linq; using System.Linq.Dynamic.Core; +using System.Threading; using System.Threading.Tasks; using Foundatio.Parsers.LuceneQueries.Nodes; using Foundatio.Parsers.LuceneQueries.Visitors; @@ -448,6 +449,9 @@ public static string TryGetNationalNumber(string phoneNumber, string regionCode public IServiceProvider GetServiceProvider() { + string sqlConnectionString = "Server=localhost;User Id=sa;Password=P@ssword1;Timeout=5;Initial Catalog=foundatio;Encrypt=False"; + SqlWaiter.Wait(sqlConnectionString); + var services = new ServiceCollection(); services.AddDbContext((_, x) => { @@ -536,6 +540,37 @@ ON ftCatalog return db; } + private static bool _checked; + private static readonly object _lock = new(); + + private static void WaitForSql( + string connectionString, + int maxRetries = 90, + int delayMs = 1000) + { + for (int i = 0; i < maxRetries; i++) + { + try + { + using var conn = new SqlConnection(connectionString); + conn.Open(); + + using var cmd = conn.CreateCommand(); + cmd.CommandText = "SELECT 1"; + cmd.ExecuteScalar(); + + return; + } + catch + { + if (i == maxRetries - 1) + throw; + + Thread.Sleep(delayMs); + } + } + } + private async Task WaitForFullTextIndexAsync(DbContext db, string catalogName, int timeoutSeconds = 30) { var end = DateTime.UtcNow.AddSeconds(timeoutSeconds); diff --git a/tests/Foundatio.Parsers.SqlQueries.Tests/SqlWaiter.cs b/tests/Foundatio.Parsers.SqlQueries.Tests/SqlWaiter.cs new file mode 100644 index 0000000..c3f151f --- /dev/null +++ b/tests/Foundatio.Parsers.SqlQueries.Tests/SqlWaiter.cs @@ -0,0 +1,63 @@ +using System; +using System.Threading; +using Microsoft.Data.SqlClient; + +namespace Foundatio.Parsers.SqlQueries.Tests; + +public static class SqlWaiter +{ + private static bool _checked; + private static readonly object _lock = new(); + + public static void Wait( + string connectionString, + TimeSpan? timeout = null, + int delayMs = 1000) + { + if (_checked) + return; + + lock (_lock) + { + if (_checked) + return; + + timeout ??= TimeSpan.FromSeconds(30); + var end = DateTime.UtcNow + timeout.Value; + + string masterCs = BuildMasterConnectionString(connectionString); + + while (DateTime.UtcNow < end) + { + try + { + using var conn = new SqlConnection(masterCs); + conn.Open(); + + using var cmd = conn.CreateCommand(); + cmd.CommandText = "SELECT 1"; + cmd.ExecuteScalar(); + + _checked = true; + return; + } + catch + { + Thread.Sleep(delayMs); + } + } + + throw new Exception("Failed to connect to SQL Server within timeout."); + } + } + + private static string BuildMasterConnectionString(string cs) + { + var builder = new SqlConnectionStringBuilder(cs) + { + InitialCatalog = "master" + }; + + return builder.ToString(); + } +} From 17d9cac995bd37d784bd193393341d38fcf3f3c0 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Wed, 19 Nov 2025 11:53:14 -0600 Subject: [PATCH 7/8] Cleanup --- .../SqlQueryParserTests.cs | 56 +------------------ .../SqlWaiter.cs | 24 ++++++++ 2 files changed, 26 insertions(+), 54 deletions(-) diff --git a/tests/Foundatio.Parsers.SqlQueries.Tests/SqlQueryParserTests.cs b/tests/Foundatio.Parsers.SqlQueries.Tests/SqlQueryParserTests.cs index 40bda30..b589cfe 100644 --- a/tests/Foundatio.Parsers.SqlQueries.Tests/SqlQueryParserTests.cs +++ b/tests/Foundatio.Parsers.SqlQueries.Tests/SqlQueryParserTests.cs @@ -79,7 +79,7 @@ public async Task CanSearchDefaultFields() sqlActual = db.Employees.Where(parser.ParsingConfig, sql).ToQueryString(); Assert.Equal(sqlExpected, sqlActual); - await WaitForFullTextIndexAsync(db, "ftCatalog"); + await SqlWaiter.WaitForFullTextIndexAsync(db, "ftCatalog"); var results = await db.Employees.Where(parser.ParsingConfig, sql).ToListAsync(); Assert.Single(results); @@ -114,7 +114,7 @@ public async Task CanSearchWithTokenizer() _logger.LogInformation(sql); Assert.Equal(sqlExpected, sqlActual); - await WaitForFullTextIndexAsync(db, "ftCatalog"); + await SqlWaiter.WaitForFullTextIndexAsync(db, "ftCatalog"); sqlActual = db.Employees.Where(parser.ParsingConfig, sql).ToQueryString(); var results = await db.Employees.Where(parser.ParsingConfig, sql).ToListAsync(); @@ -540,58 +540,6 @@ ON ftCatalog return db; } - private static bool _checked; - private static readonly object _lock = new(); - - private static void WaitForSql( - string connectionString, - int maxRetries = 90, - int delayMs = 1000) - { - for (int i = 0; i < maxRetries; i++) - { - try - { - using var conn = new SqlConnection(connectionString); - conn.Open(); - - using var cmd = conn.CreateCommand(); - cmd.CommandText = "SELECT 1"; - cmd.ExecuteScalar(); - - return; - } - catch - { - if (i == maxRetries - 1) - throw; - - Thread.Sleep(delayMs); - } - } - } - - private async Task WaitForFullTextIndexAsync(DbContext db, string catalogName, int timeoutSeconds = 30) - { - var end = DateTime.UtcNow.AddSeconds(timeoutSeconds); - - while (DateTime.UtcNow < end) - { - string sql = "SELECT FULLTEXTCATALOGPROPERTY(@catalogName, 'PopulateStatus') AS Value"; - - int status = await db.Database - .SqlQueryRaw(sql, new SqlParameter("@catalogName", catalogName)) - .SingleAsync(); - - if (status == 0) - return; - - await Task.Delay(500); - } - - throw new TimeoutException($"Full-text catalog '{catalogName}' didn't finish populating in time."); - } - private async Task ParseAndValidateQuery(string query, string expected, bool isValid) { #if ENABLE_TRACING diff --git a/tests/Foundatio.Parsers.SqlQueries.Tests/SqlWaiter.cs b/tests/Foundatio.Parsers.SqlQueries.Tests/SqlWaiter.cs index c3f151f..f0ca6b5 100644 --- a/tests/Foundatio.Parsers.SqlQueries.Tests/SqlWaiter.cs +++ b/tests/Foundatio.Parsers.SqlQueries.Tests/SqlWaiter.cs @@ -1,6 +1,8 @@ using System; using System.Threading; +using System.Threading.Tasks; using Microsoft.Data.SqlClient; +using Microsoft.EntityFrameworkCore; namespace Foundatio.Parsers.SqlQueries.Tests; @@ -51,6 +53,28 @@ public static void Wait( } } + + public static async Task WaitForFullTextIndexAsync(DbContext db, string catalogName, int timeoutSeconds = 30) + { + var end = DateTime.UtcNow.AddSeconds(timeoutSeconds); + + while (DateTime.UtcNow < end) + { + string sql = "SELECT FULLTEXTCATALOGPROPERTY(@catalogName, 'PopulateStatus') AS Value"; + + int status = await db.Database + .SqlQueryRaw(sql, new SqlParameter("@catalogName", catalogName)) + .SingleAsync(); + + if (status == 0) + return; + + await Task.Delay(500); + } + + throw new TimeoutException($"Full-text catalog '{catalogName}' didn't finish populating in time."); + } + private static string BuildMasterConnectionString(string cs) { var builder = new SqlConnectionStringBuilder(cs) From 4f60443a3df90fc55270ed45555a247b927c9445 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Mon, 24 Nov 2025 17:05:56 -0600 Subject: [PATCH 8/8] Fix bugs with full-text --- .../Extensions/SqlNodeExtensions.cs | 52 ++++++++++++++----- .../Visitors/SqlQueryVisitorContext.cs | 26 ++++++++++ .../SampleContext.cs | 12 ++++- .../SqlQueryParserTests.cs | 43 +++++++++++++-- 4 files changed, 115 insertions(+), 18 deletions(-) diff --git a/src/Foundatio.Parsers.SqlQueries/Extensions/SqlNodeExtensions.cs b/src/Foundatio.Parsers.SqlQueries/Extensions/SqlNodeExtensions.cs index 70f5c2c..756ba08 100644 --- a/src/Foundatio.Parsers.SqlQueries/Extensions/SqlNodeExtensions.cs +++ b/src/Foundatio.Parsers.SqlQueries/Extensions/SqlNodeExtensions.cs @@ -164,10 +164,12 @@ public static string ToDynamicLinqString(this TermNode node, ISqlQueryVisitorCon var searchTerm = kvp.Value; var tokens = kvp.Value.Tokens ?? [kvp.Value.Term]; var (fieldPrefix, fieldSuffix) = kvp.Key.GetFieldPrefixAndSuffix(); + var (scopePrefix, argumentPrefix) = SplitFieldPrefix(kvp.Key, fieldPrefix); if (searchTerm.Operator == SqlSearchOperator.Equals) { - builder.Append(fieldPrefix); + builder.Append(scopePrefix); + builder.Append(argumentPrefix); builder.Append(kvp.Key.Name); builder.Append(" in ("); for (int i = 0; i < tokens.Count; i++) @@ -185,11 +187,13 @@ public static string ToDynamicLinqString(this TermNode node, ISqlQueryVisitorCon tokens.ForEach((token, i) => { builder.Append(i.IsFirst ? "(" : " OR "); - builder.Append(fieldPrefix); - if (context.FullTextFields.Contains(kvp.Key.Name, StringComparer.OrdinalIgnoreCase)) + builder.Append(scopePrefix); + + if (context.FullTextFields.Contains(kvp.Key.FullName, StringComparer.OrdinalIgnoreCase)) { builder.Append("FTS.Contains("); + builder.Append(argumentPrefix); builder.Append(kvp.Key.Name); builder.Append(", "); AppendField(builder, kvp.Key, token, context); @@ -197,6 +201,7 @@ public static string ToDynamicLinqString(this TermNode node, ISqlQueryVisitorCon } else { + builder.Append(argumentPrefix); builder.Append(kvp.Key.Name); builder.Append(".Contains("); AppendField(builder, kvp.Key, token, context); @@ -213,11 +218,12 @@ public static string ToDynamicLinqString(this TermNode node, ISqlQueryVisitorCon tokens.ForEach((token, i) => { builder.Append(i.IsFirst ? "(" : " OR "); - builder.Append(fieldPrefix); + builder.Append(scopePrefix); - if (context.FullTextFields.Contains(kvp.Key.Name, StringComparer.OrdinalIgnoreCase)) + if (context.FullTextFields.Contains(kvp.Key.FullName, StringComparer.OrdinalIgnoreCase)) { builder.Append("FTS.Contains("); + builder.Append(argumentPrefix); builder.Append(kvp.Key.Name); builder.Append(", "); AppendField(builder, kvp.Key, "\\\"" + token + "*\\\"", context); @@ -225,6 +231,7 @@ public static string ToDynamicLinqString(this TermNode node, ISqlQueryVisitorCon } else { + builder.Append(argumentPrefix); builder.Append(kvp.Key.Name); builder.Append(".StartsWith("); AppendField(builder, kvp.Key, token, context); @@ -246,6 +253,7 @@ public static string ToDynamicLinqString(this TermNode node, ISqlQueryVisitorCon var field = GetFieldInfo(context.Fields, node.Field); var (fieldPrefix, fieldSuffix) = field.GetFieldPrefixAndSuffix(); + var (scopePrefix, argumentPrefix) = SplitFieldPrefix(field, fieldPrefix); var searchOperator = SqlSearchOperator.Equals; if (node.Term.StartsWith("*") && node.Term.EndsWith("*")) searchOperator = SqlSearchOperator.Contains; @@ -257,7 +265,8 @@ public static string ToDynamicLinqString(this TermNode node, ISqlQueryVisitorCon if (searchOperator == SqlSearchOperator.Equals) { - builder.Append(fieldPrefix); + builder.Append(scopePrefix); + builder.Append(argumentPrefix); builder.Append(field.Name); builder.Append(" = "); AppendField(builder, field, node.Term, context); @@ -265,11 +274,12 @@ public static string ToDynamicLinqString(this TermNode node, ISqlQueryVisitorCon } else if (searchOperator == SqlSearchOperator.Contains) { - builder.Append(fieldPrefix); + builder.Append(scopePrefix); - if (context.FullTextFields.Contains(field.Name, StringComparer.OrdinalIgnoreCase)) + if (context.FullTextFields.Contains(field.FullName, StringComparer.OrdinalIgnoreCase)) { builder.Append("FTS.Contains("); + builder.Append(argumentPrefix); builder.Append(field.Name); builder.Append(", "); AppendField(builder, field, node.Term, context); @@ -277,6 +287,7 @@ public static string ToDynamicLinqString(this TermNode node, ISqlQueryVisitorCon } else { + builder.Append(argumentPrefix); builder.Append(field.Name); builder.Append(".Contains("); AppendField(builder, field, node.Term, context); @@ -287,11 +298,12 @@ public static string ToDynamicLinqString(this TermNode node, ISqlQueryVisitorCon } else { - builder.Append(fieldPrefix); + builder.Append(scopePrefix); - if (context.FullTextFields.Contains(field.Name, StringComparer.OrdinalIgnoreCase)) + if (context.FullTextFields.Contains(field.FullName, StringComparer.OrdinalIgnoreCase)) { builder.Append("FTS.Contains("); + builder.Append(argumentPrefix); builder.Append(field.Name); builder.Append(", "); AppendField(builder, field, "\\\"" + node.Term + "*\\\"", context); @@ -299,6 +311,7 @@ public static string ToDynamicLinqString(this TermNode node, ISqlQueryVisitorCon } else { + builder.Append(argumentPrefix); builder.Append(field.Name); builder.Append(".Contains("); AppendField(builder, field, node.Term, context); @@ -329,6 +342,7 @@ public static string ToDynamicLinqString(this TermRangeNode node, ISqlQueryVisit context.AddValidationError("Field must be a number, money or date for term range queries."); var (fieldPrefix, fieldSuffix) = field.GetFieldPrefixAndSuffix(); + var (scopePrefix, argumentPrefix) = SplitFieldPrefix(field, fieldPrefix); var builder = new StringBuilder(); @@ -340,7 +354,8 @@ public static string ToDynamicLinqString(this TermRangeNode node, ISqlQueryVisit if (node.Min != null) { - builder.Append(fieldPrefix); + builder.Append(scopePrefix); + builder.Append(argumentPrefix); builder.Append(field.Name); builder.Append(node.MinInclusive == true ? " >= " : " > "); AppendField(builder, field, node.Min, context); @@ -352,7 +367,8 @@ public static string ToDynamicLinqString(this TermRangeNode node, ISqlQueryVisit if (node.Max != null) { - builder.Append(fieldPrefix); + builder.Append(scopePrefix); + builder.Append(argumentPrefix); builder.Append(field.Name); builder.Append(node.MaxInclusive == true ? " <= " : " < "); AppendField(builder, field, node.Max, context); @@ -387,6 +403,18 @@ public static EntityFieldInfo GetFieldInfo(List fields, string new EntityFieldInfo { Name = field, FullName = field }; } + private static (string scopePrefix, string argumentPrefix) SplitFieldPrefix(EntityFieldInfo field, string fieldPrefix) + { + string navigationPrefix = field.GetNavigationPrefix(); + if (String.IsNullOrEmpty(navigationPrefix)) + return (fieldPrefix, String.Empty); + + if (!String.IsNullOrEmpty(fieldPrefix) && fieldPrefix.EndsWith(navigationPrefix, StringComparison.Ordinal)) + fieldPrefix = fieldPrefix.Substring(0, fieldPrefix.Length - navigationPrefix.Length); + + return (fieldPrefix, navigationPrefix); + } + private static void AppendField(StringBuilder builder, EntityFieldInfo field, string term, ISqlQueryVisitorContext context) { if (field == null) diff --git a/src/Foundatio.Parsers.SqlQueries/Visitors/SqlQueryVisitorContext.cs b/src/Foundatio.Parsers.SqlQueries/Visitors/SqlQueryVisitorContext.cs index e1e73fe..ab8c232 100644 --- a/src/Foundatio.Parsers.SqlQueries/Visitors/SqlQueryVisitorContext.cs +++ b/src/Foundatio.Parsers.SqlQueries/Visitors/SqlQueryVisitorContext.cs @@ -86,6 +86,32 @@ public override bool Equals(object obj) return (prefix.ToString(), suffix.ToString()); } + + public string GetNavigationPrefix() + { + if (Parent == null) + return String.Empty; + + var stack = new Stack(); + var current = Parent; + while (current != null && !current.IsCollection) + { + stack.Push(current); + current = current.Parent; + } + + if (stack.Count == 0) + return String.Empty; + + var builder = new StringBuilder(); + while (stack.Count > 0) + { + var field = stack.Pop(); + builder.Append(field.Name).Append('.'); + } + + return builder.ToString(); + } } public class SearchTerm diff --git a/tests/Foundatio.Parsers.SqlQueries.Tests/SampleContext.cs b/tests/Foundatio.Parsers.SqlQueries.Tests/SampleContext.cs index b8bc04a..0b398e3 100644 --- a/tests/Foundatio.Parsers.SqlQueries.Tests/SampleContext.cs +++ b/tests/Foundatio.Parsers.SqlQueries.Tests/SampleContext.cs @@ -19,7 +19,14 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) base.OnModelCreating(modelBuilder); // Employee - modelBuilder.Entity().HasIndex(e => new { e.FullName, e.Title }); + var employee = modelBuilder.Entity(); + employee.HasIndex(e => new { e.FullName, e.Title }); + employee.HasOne(e => e.CurrentCompany) + .WithMany() + .HasForeignKey(e => e.CurrentCompanyId) + .OnDelete(DeleteBehavior.Restrict); + employee.HasMany(e => e.Companies) + .WithMany(c => c.Employees); // Company modelBuilder.Entity().HasIndex(e => new { e.Name }); @@ -56,6 +63,8 @@ public class Employee public string NationalPhoneNumber { get; set; } public string Title { get; set; } public int Salary { get; set; } + public int? CurrentCompanyId { get; set; } + public Company CurrentCompany { get; set; } public List Companies { get; set; } public List DataValues { get; set; } public TimeOnly HappyHour { get; set; } @@ -69,6 +78,7 @@ public class Company public int Id { get; set; } public string Name { get; set; } public string Description { get; set; } + public string Location { get; set; } public List Employees { get; set; } public List DataDefinitions { get; set; } } diff --git a/tests/Foundatio.Parsers.SqlQueries.Tests/SqlQueryParserTests.cs b/tests/Foundatio.Parsers.SqlQueries.Tests/SqlQueryParserTests.cs index b589cfe..8930869 100644 --- a/tests/Foundatio.Parsers.SqlQueries.Tests/SqlQueryParserTests.cs +++ b/tests/Foundatio.Parsers.SqlQueries.Tests/SqlQueryParserTests.cs @@ -2,13 +2,11 @@ using System.ComponentModel.DataAnnotations; using System.Linq; using System.Linq.Dynamic.Core; -using System.Threading; using System.Threading.Tasks; using Foundatio.Parsers.LuceneQueries.Nodes; using Foundatio.Parsers.LuceneQueries.Visitors; using Foundatio.Parsers.SqlQueries.Visitors; using Foundatio.Xunit; -using Microsoft.Data.SqlClient; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Infrastructure; using Microsoft.Extensions.DependencyInjection; @@ -337,6 +335,30 @@ public async Task CanUseCollectionDefaultFieldsWithNestedDepth() Assert.Equal(sqlExpected, sqlActual); } + [Fact] + public async Task CanUseCurrentCompanyFullText() + { + var sp = GetServiceProvider(); + await using var db = await GetSampleContextWithDataAsync(sp); + var parser = sp.GetRequiredService(); + parser.Configuration.SetDefaultFields(["CurrentCompany.Name", "CurrentCompany.Location"]); + + var context = parser.GetContext(db.Employees.EntityType); + + string sqlExpected = db.Employees.Where(e => EF.Functions.Contains(e.CurrentCompany.Name, "\"acme*\"") || EF.Functions.Contains(e.CurrentCompany.Location, "\"acme*\"")).ToQueryString(); + + await SqlWaiter.WaitForFullTextIndexAsync(db, "ftCatalog"); + + string sql = await parser.ToDynamicLinqAsync("acme", context); + Assert.Contains("FTS.Contains(CurrentCompany.Name", sql); + Assert.Contains("FTS.Contains(CurrentCompany.Location", sql); + + string sqlActual = db.Employees.Where(parser.ParsingConfig, sql).ToQueryString(); + Assert.Equal(sqlExpected, sqlActual); + var results = await db.Employees.Where(parser.ParsingConfig, sql).ToListAsync(); + Assert.Equal(2, results.Count); + } + [Fact] public async Task CanUseNavigationFields() { @@ -461,7 +483,7 @@ public IServiceProvider GetServiceProvider() parser.Configuration.UseEntityTypePropertyFilter(p => p.Name != nameof(Company.Description)); parser.Configuration.AddQueryVisitor(new DynamicFieldVisitor()); parser.Configuration.SetDefaultFields(["FullName"], SqlSearchOperator.Contains); - parser.Configuration.SetFullTextFields(["Name", "FullName", "Title", "NationalPhoneNumber"]); + parser.Configuration.SetFullTextFields(["Name", "FullName", "Title", "NationalPhoneNumber", "CurrentCompany.Name", "CurrentCompany.Location", "Companies.Name", "Companies.Location"]); services.AddSingleton(parser); return services.BuildServiceProvider(); } @@ -496,7 +518,8 @@ public async Task GetSampleContextWithDataAsync(IServiceProvider Salary = 80_000, Birthday = new DateOnly(1980, 1, 1), DataValues = [new() { Definition = company.DataDefinitions[0], NumberValue = 30 }], - Companies = [company] + Companies = [company], + CurrentCompany = company }); db.Employees.Add(new Employee { @@ -507,7 +530,8 @@ public async Task GetSampleContextWithDataAsync(IServiceProvider Salary = 90_000, Birthday = new DateOnly(1972, 11, 6), DataValues = [new() { Definition = company.DataDefinitions[0], NumberValue = 23 }], - Companies = [company] + Companies = [company], + CurrentCompany = company }); await db.SaveChangesAsync(); @@ -535,6 +559,15 @@ Title LANGUAGE 1033 ) KEY INDEX PK_Employees ON ftCatalog + WITH (CHANGE_TRACKING = AUTO, STOPLIST = SYSTEM); + + CREATE FULLTEXT INDEX ON Companies + ( + Name LANGUAGE 1033, + Location LANGUAGE 1033 + ) + KEY INDEX PK_Companies + ON ftCatalog WITH (CHANGE_TRACKING = AUTO, STOPLIST = SYSTEM);"); return db;