Skip to content

Commit dff0677

Browse files
ejsmithejsmith
committed
Adding support for full text search
1 parent b37d910 commit dff0677

File tree

10 files changed

+154
-14
lines changed

10 files changed

+154
-14
lines changed

‎Dockerfile‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
FROM mcr.microsoft.com/mssql/server:2022-latest
2+
3+
ARG SSID_PID=Developer
4+
5+
ENV ACCEPT_EULA=Y
6+
ENV SSID_PID=${SSID_PID}
7+
ENV DEBIAN_FRONTEND=noninteractive
8+
ENV DEBCONF_NONINTERACTIVE_SEEN=true
9+
10+
USER root
11+
12+
RUN apt-get update && \
13+
apt-get upgrade -y && \
14+
apt-get install -yq gnupg gnupg2 gnupg1 curl apt-transport-https && \
15+
curl https://packages.microsoft.com/keys/microsoft.asc -o /var/opt/mssql/ms-key.cer && \
16+
gpg --dearmor -o /etc/apt/trusted.gpg.d/microsoft.gpg /var/opt/mssql/ms-key.cer && \
17+
curl https://packages.microsoft.com/config/ubuntu/22.04/mssql-server-2022.list -o /etc/apt/sources.list.d/mssql-server-2022.list && \
18+
apt-get update && \
19+
apt-get install -y mssql-server-fts && \
20+
apt-get clean && \
21+
rm -rf /var/lib/apt/lists
22+
23+
ENTRYPOINT [ "/opt/mssql/bin/sqlservr" ]

‎docker-compose.yml‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,9 @@ services:
3030
test: curl --write-out 'HTTP %{http_code}' --fail --silent --output /dev/null http://localhost:5601/api/status
3131

3232
sqlserver:
33-
image: mcr.microsoft.com/mssql/server:2025-latest
33+
build:
34+
context: .
35+
dockerfile: Dockerfile
3436
ports:
3537
- "1433:1433" # login with sa:P@ssword1
3638
environment:

‎src/Foundatio.Parsers.SqlQueries/Extensions/SqlNodeExtensions.cs‎

Lines changed: 35 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -142,7 +142,7 @@ public static string ToDynamicLinqString(this TermNode node, ISqlQueryVisitorCon
142142
{
143143
FieldInfo = fieldInfo,
144144
Term = node.Term,
145-
Operator = SqlSearchOperator.StartsWith
145+
Operator = context.DefaultSearchOperator
146146
};
147147
fieldTerms[fieldInfo] = searchTerm;
148148
}
@@ -186,10 +186,23 @@ public static string ToDynamicLinqString(this TermNode node, ISqlQueryVisitorCon
186186
{
187187
builder.Append(i.IsFirst ? "(" : " OR ");
188188
builder.Append(fieldPrefix);
189-
builder.Append(kvp.Key.Name);
190-
builder.Append(".Contains(");
191-
AppendField(builder, kvp.Key, token, context);
192-
builder.Append(")");
189+
190+
if (context.FullTextSearchEnabled)
191+
{
192+
builder.Append("FTS.Contains(");
193+
builder.Append(kvp.Key.Name);
194+
builder.Append(", ");
195+
AppendField(builder, kvp.Key, token, context);
196+
builder.Append(")");
197+
}
198+
else
199+
{
200+
builder.Append(kvp.Key.Name);
201+
builder.Append(".Contains(");
202+
AppendField(builder, kvp.Key, token, context);
203+
builder.Append(")");
204+
}
205+
193206
builder.Append(fieldSuffix);
194207
if (i.IsLast)
195208
builder.Append(")");
@@ -201,10 +214,23 @@ public static string ToDynamicLinqString(this TermNode node, ISqlQueryVisitorCon
201214
{
202215
builder.Append(i.IsFirst ? "(" : " OR ");
203216
builder.Append(fieldPrefix);
204-
builder.Append(kvp.Key.Name);
205-
builder.Append(".StartsWith(");
206-
AppendField(builder, kvp.Key, token, context);
207-
builder.Append(")");
217+
218+
if (context.FullTextSearchEnabled)
219+
{
220+
builder.Append("FTS.Contains(");
221+
builder.Append(kvp.Key.Name);
222+
builder.Append(", ");
223+
AppendField(builder, kvp.Key, token + "*", context);
224+
builder.Append(")");
225+
}
226+
else
227+
{
228+
builder.Append(kvp.Key.Name);
229+
builder.Append(".StartsWith(");
230+
AppendField(builder, kvp.Key, token, context);
231+
builder.Append(")");
232+
}
233+
208234
builder.Append(fieldSuffix);
209235
if (i.IsLast)
210236
builder.Append(")");

‎src/Foundatio.Parsers.SqlQueries/Foundatio.Parsers.SqlQueries.csproj‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
<TargetFrameworks>net10.0;</TargetFrameworks>
55
</PropertyGroup>
66
<ItemGroup>
7-
<PackageReference Include="Microsoft.EntityFrameworkCore" Version="10.0.0" />
7+
<PackageReference Include="Microsoft.EntityFrameworkCore.SqlServer" Version="10.0.0" />
88
<PackageReference Include="Exceptionless.DateTimeExtensions" Version="4.0.1" />
99
<PackageReference Include="System.Linq.Dynamic.Core" Version="1.6.10" />
1010
</ItemGroup>

‎src/Foundatio.Parsers.SqlQueries/SqlQueryParser.cs‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,16 @@
33
using System.Collections.Generic;
44
using System.ComponentModel.DataAnnotations;
55
using System.Linq;
6+
using System.Linq.Dynamic.Core;
7+
using System.Linq.Dynamic.Core.CustomTypeProviders;
68
using System.Threading.Tasks;
79
using Foundatio.Parsers.LuceneQueries;
810
using Foundatio.Parsers.LuceneQueries.Extensions;
911
using Foundatio.Parsers.LuceneQueries.Nodes;
1012
using Foundatio.Parsers.LuceneQueries.Visitors;
1113
using Foundatio.Parsers.SqlQueries.Extensions;
1214
using Foundatio.Parsers.SqlQueries.Visitors;
15+
using Microsoft.EntityFrameworkCore;
1316
using Microsoft.EntityFrameworkCore.Metadata;
1417
using Pegasus.Common;
1518

@@ -25,6 +28,10 @@ public SqlQueryParser(Action<SqlQueryParserConfiguration> configure = null)
2528
}
2629

2730
public SqlQueryParserConfiguration Configuration { get; }
31+
public ParsingConfig ParsingConfig { get; } = new()
32+
{
33+
CustomTypeProvider = new DynamicLinqTypeProvider()
34+
};
2835

2936
public override async Task<IQueryNode> ParseAsync(string query, IQueryVisitorContext context = null)
3037
{
@@ -217,6 +224,18 @@ private void SetupQueryVisitorContextDefaults(IQueryVisitorContext context)
217224
sqlContext.SearchTokenizer = Configuration.SearchTokenizer;
218225
sqlContext.DateTimeParser = Configuration.DateTimeParser;
219226
sqlContext.DateOnlyParser = Configuration.DateOnlyParser;
227+
sqlContext.DefaultSearchOperator = Configuration.DefaultFieldsSearchOperator;
228+
sqlContext.FullTextSearchEnabled = Configuration.FullTextSearchEnabled;
220229
}
221230
}
222231
}
232+
233+
public static class FTS
234+
{
235+
public static bool Contains(string propertyValue, string searchTerm)
236+
{
237+
return EF.Functions.Contains(propertyValue, searchTerm);
238+
}
239+
}
240+
241+
public class DynamicLinqTypeProvider() : DefaultDynamicLinqCustomTypeProvider(ParsingConfig.Default, [ typeof(EF), typeof(FTS) ]);

‎src/Foundatio.Parsers.SqlQueries/SqlQueryParserConfiguration.cs‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,8 @@ public SqlQueryParserConfiguration()
2323

2424
public ILoggerFactory LoggerFactory { get; private set; } = NullLoggerFactory.Instance;
2525
public string[] DefaultFields { get; private set; }
26+
public SqlSearchOperator DefaultFieldsSearchOperator { get; private set; } = SqlSearchOperator.StartsWith;
27+
public bool FullTextSearchEnabled { get; private set; } = false;
2628

2729
public int MaxFieldDepth { get; private set; } = 10;
2830
public QueryFieldResolver FieldResolver { get; private set; }
@@ -46,9 +48,10 @@ public SqlQueryParserConfiguration SetLoggerFactory(ILoggerFactory loggerFactory
4648
return this;
4749
}
4850

49-
public SqlQueryParserConfiguration SetDefaultFields(string[] fields)
51+
public SqlQueryParserConfiguration SetDefaultFields(string[] fields, SqlSearchOperator op = SqlSearchOperator.StartsWith)
5052
{
5153
DefaultFields = fields;
54+
DefaultFieldsSearchOperator = op;
5255
return this;
5356
}
5457

@@ -58,6 +61,12 @@ public SqlQueryParserConfiguration SetSearchTokenizer(Action<SearchTerm> tokeniz
5861
return this;
5962
}
6063

64+
public SqlQueryParserConfiguration UseFullTextSearch(bool useFullTextSearch = true)
65+
{
66+
FullTextSearchEnabled = useFullTextSearch;
67+
return this;
68+
}
69+
6170
public SqlQueryParserConfiguration SetDateTimeParser(Func<string, string> dateTimeParser)
6271
{
6372
DateTimeParser = dateTimeParser;

‎src/Foundatio.Parsers.SqlQueries/Visitors/ISqlQueryVisitorContext.cs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@ namespace Foundatio.Parsers.SqlQueries.Visitors;
77
public interface ISqlQueryVisitorContext : IQueryVisitorContext
88
{
99
List<EntityFieldInfo> Fields { get; set; }
10+
SqlSearchOperator DefaultSearchOperator { get; set; }
11+
bool FullTextSearchEnabled { get; set; }
1012
Action<SearchTerm> SearchTokenizer { get; set; }
1113
Func<string, string> DateTimeParser { get; set; }
1214
Func<string, string> DateOnlyParser { get; set; }

‎src/Foundatio.Parsers.SqlQueries/Visitors/SqlQueryVisitorContext.cs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,8 @@ namespace Foundatio.Parsers.SqlQueries.Visitors;
1010
public class SqlQueryVisitorContext : QueryVisitorContext, ISqlQueryVisitorContext
1111
{
1212
public List<EntityFieldInfo> Fields { get; set; }
13+
public SqlSearchOperator DefaultSearchOperator { get; set; } = SqlSearchOperator.StartsWith;
14+
public bool FullTextSearchEnabled { get; set; } = false;
1315
public Action<SearchTerm> SearchTokenizer { get; set; } = static _ => { };
1416
public Func<string, string> DateTimeParser { get; set; }
1517
public Func<string, string> DateOnlyParser { get; set; }

‎tests/Foundatio.Parsers.SqlQueries.Tests/SampleContext.cs‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
using System;
22
using System.Collections.Generic;
3+
using System.Linq;
34
using Microsoft.EntityFrameworkCore;
5+
using Microsoft.EntityFrameworkCore.Query.SqlExpressions;
46

57
namespace Foundatio.Parsers.SqlQueries.Tests;
68

@@ -34,6 +36,15 @@ protected override void OnModelCreating(ModelBuilder modelBuilder)
3436
modelBuilder.Entity<DataValue>().Property(e => e.BooleanValue).IsSparse();
3537
modelBuilder.Entity<DataValue>().Property(e => e.NumberValue).HasColumnType("decimal").HasPrecision(15, 3).IsSparse();
3638
modelBuilder.Entity<DataValue>().HasIndex(e => new { e.StringValue, e.DateValue, e.MoneyValue, e.BooleanValue, e.NumberValue });
39+
40+
modelBuilder.HasDbFunction(typeof(FTS).GetMethod(nameof(FTS.Contains))!)
41+
.HasTranslation(args => new SqlFunctionExpression(
42+
"CONTAINS",
43+
args,
44+
true,
45+
args.Select(a => false).ToList(),
46+
typeof(bool),
47+
null));
3748
}
3849
}
3950

‎tests/Foundatio.Parsers.SqlQueries.Tests/SqlQueryParserTests.cs‎

Lines changed: 48 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -377,22 +377,39 @@ public async Task CanUseSkipNavigationFields()
377377
Assert.Single(companies);
378378
}
379379

380+
[Fact]
381+
public async Task CanUseLike()
382+
{
383+
var sp = GetServiceProvider();
384+
await using var db = await GetSampleContextWithDataAsync(sp);
385+
var parser = sp.GetRequiredService<SqlQueryParser>();
386+
387+
var context = parser.GetContext(db.Employees.EntityType);
388+
string sqlExpected = db.Employees.Where(e => EF.Functions.Contains(e.FullName, "john")).ToQueryString();
389+
string sqlActual = db.Employees.Where(parser.ParsingConfig, """FTS.Contains(FullName, "john")""").ToQueryString();
390+
Assert.Equal(sqlExpected, sqlActual);
391+
string sql = await parser.ToDynamicLinqAsync("john", context);
392+
sqlActual = db.Employees.Where(parser.ParsingConfig, sql).ToQueryString();
393+
Assert.Equal(sqlExpected, sqlActual);
394+
}
395+
380396
[Fact]
381397
public async Task CanGenerateSql()
382398
{
383399
var sp = GetServiceProvider();
384400
await using var db = await GetSampleContextWithDataAsync(sp);
385401
var parser = sp.GetRequiredService<SqlQueryParser>();
386402

403+
var efFunctions = EF.Functions;
387404
var context = parser.GetContext(db.Employees.EntityType);
388405
context.Fields.Add(new EntityFieldInfo { Name = "age", FullName = "age", IsNumber = true, Data = { { "DataDefinitionId", 1 } } });
389406
context.ValidationOptions.AllowedFields.Add("age");
390407

391408
string sqlExpected = db.Employees.Where(e => e.Companies.Any(c => c.Name == "acme") && e.DataValues.Any(dv => dv.DataDefinitionId == 1 && dv.NumberValue == 30)).ToQueryString();
392409
string sqlActual = db.Employees.Where("""Companies.Any(Name = "acme") AND DataValues.Any(DataDefinitionId = 1 AND NumberValue = 30) """).ToQueryString();
393410
Assert.Equal(sqlExpected, sqlActual);
394-
string sql = await parser.ToDynamicLinqAsync("companies.name:acme age:30", context);
395-
sqlActual = db.Employees.Where(sql).ToQueryString();
411+
string sql = await parser.ToDynamicLinqAsync("EF.Contains(@0, companies.name, @1)companies.name:acme age:30", context);
412+
sqlActual = db.Employees.Where(sql, efFunctions).ToQueryString();
396413
Assert.Equal(sqlExpected, sqlActual);
397414

398415
var q = db.Employees.AsNoTracking();
@@ -433,6 +450,8 @@ public IServiceProvider GetServiceProvider()
433450
var parser = new SqlQueryParser();
434451
parser.Configuration.UseEntityTypePropertyFilter(p => p.Name != nameof(Company.Description));
435452
parser.Configuration.AddQueryVisitor(new DynamicFieldVisitor());
453+
parser.Configuration.SetDefaultFields(["FullName"], SqlSearchOperator.Contains);
454+
parser.Configuration.UseFullTextSearch();
436455
services.AddSingleton(parser);
437456
return services.BuildServiceProvider();
438457
}
@@ -482,6 +501,33 @@ public async Task<SampleContext> GetSampleContextWithDataAsync(IServiceProvider
482501
});
483502
await db.SaveChangesAsync();
484503

504+
var result = await db.Database.ExecuteSqlRawAsync(
505+
@"IF FULLTEXTSERVICEPROPERTY('IsFullTextInstalled') != 1
506+
BEGIN
507+
RAISERROR('Full-Text Search is not installed', 16, 1);
508+
END");
509+
510+
result = await db.Database.ExecuteSqlRawAsync(
511+
@"IF NOT EXISTS (SELECT * FROM sys.fulltext_catalogs WHERE name = 'ftCatalog')
512+
BEGIN
513+
CREATE FULLTEXT CATALOG ftCatalog AS DEFAULT;
514+
END");
515+
516+
result = await db.Database.ExecuteSqlRawAsync(
517+
@"IF EXISTS (SELECT * FROM sys.fulltext_indexes WHERE object_id = OBJECT_ID('Employees'))
518+
BEGIN
519+
DROP FULLTEXT INDEX ON Employees;
520+
END");
521+
522+
result = await db.Database.ExecuteSqlRawAsync(
523+
@"CREATE FULLTEXT INDEX ON Employees
524+
(
525+
FullName LANGUAGE 1033
526+
)
527+
KEY INDEX PK_Employees
528+
ON ftCatalog
529+
WITH (CHANGE_TRACKING = AUTO, STOPLIST = SYSTEM);");
530+
485531
return db;
486532
}
487533

0 commit comments

Comments
 (0)