Skip to content

Commit 9774483

Browse files
isaacvandoisaacvando
authored
feat: Add index management (#69)
1 parent 4990df9 commit 9774483

9 files changed

Lines changed: 823 additions & 2 deletions

‎AGENTS.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
# AGENTS.md
2+
3+
## Testing
4+
5+
Use `dotnet test -f net10.0` when verifying your changes. Do not run `dotnet build`.

‎README.md‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -71,9 +71,10 @@ If you need commercial support, please [contact the ParadeDB team](mailto:sales@
7171

7272
## Acknowledgments
7373

74-
We would like to thank the following members of the Entity Framework Core community for the initial implementation of this project:
74+
We would like to thank the following members of the Entity Framework Core community:
7575

76-
- [Nandor Krizbai](https://github.com/nandor23) - .NET developer
76+
- [Nandor Krizbai](https://github.com/nandor23) - for the initial implementation of this project
77+
- [Daniel Oliveira](https://github.com/daniel3303) - for implementing [ParadeDbEntityFrameworkCore](https://github.com/daniel3303/ParadeDbEntityFrameworkCore) which inspired our indexing implementation
7778

7879
## License
7980

‎src/Extensions/ParadeDbIndexBuilderExtensions.cs‎

Lines changed: 172 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,172 @@
1+
using System.Linq.Expressions;
2+
using Microsoft.EntityFrameworkCore;
3+
using Microsoft.EntityFrameworkCore.Metadata.Builders;
4+
using ParadeDB.EntityFrameworkCore.Internal.Metadata;
5+
6+
namespace ParadeDB.EntityFrameworkCore.Extensions;
7+
8+
public static class ParadeDbIndexBuilderExtensions
9+
{
10+
public static Bm25IndexBuilder<TEntity> HasBm25Index<TEntity>(
11+
this EntityTypeBuilder<TEntity> entityTypeBuilder,
12+
string name,
13+
Expression<Func<TEntity, object?>> keyExpression
14+
)
15+
where TEntity : class
16+
{
17+
var keyProperty = GetPropertyName(keyExpression);
18+
var indexBuilder = entityTypeBuilder.HasIndex(keyExpression).HasDatabaseName(name);
19+
20+
indexBuilder.HasAnnotation(ParadeDbAnnotationNames.Bm25KeyProperty, keyProperty);
21+
indexBuilder.HasAnnotation(
22+
ParadeDbAnnotationNames.Bm25FieldProperties,
23+
new[] { keyProperty }
24+
);
25+
// BM25FieldKinds is used to track if each index field is an EF Core property or a SQL expression
26+
// so that it can be rendered appropriately
27+
indexBuilder.HasAnnotation(ParadeDbAnnotationNames.Bm25FieldKinds, new[] { "property" });
28+
indexBuilder.HasAnnotation(ParadeDbAnnotationNames.Bm25FieldTokenizers, new[] { "" });
29+
indexBuilder.HasAnnotation(ParadeDbAnnotationNames.Bm25FieldAliases, new[] { "" });
30+
31+
return new Bm25IndexBuilder<TEntity>(indexBuilder);
32+
}
33+
34+
internal static string GetPropertyName<TEntity, TProperty>(
35+
Expression<Func<TEntity, TProperty>> propertyExpression
36+
)
37+
{
38+
var body = propertyExpression.Body is UnaryExpression unary
39+
? unary.Operand
40+
: propertyExpression.Body;
41+
42+
return body is MemberExpression member
43+
? member.Member.Name
44+
: throw new ArgumentException("The BM25 key expression must be a property access.");
45+
}
46+
}
47+
48+
public record FieldAlias(string name) { }
49+
50+
// We use a custom index builder class instead of IndexBuilder directly to make it clear that not all
51+
// normal index creation operations are supported here (e.g. `IsUnique`)
52+
public sealed class Bm25IndexBuilder<TEntity>
53+
where TEntity : class
54+
{
55+
private readonly IndexBuilder<TEntity> _indexBuilder;
56+
57+
internal Bm25IndexBuilder(IndexBuilder<TEntity> indexBuilder)
58+
{
59+
_indexBuilder = indexBuilder;
60+
}
61+
62+
public Bm25IndexBuilder<TEntity> HasField<TProperty>(
63+
Expression<Func<TEntity, TProperty>> propertyExpression
64+
)
65+
{
66+
AddField(
67+
ParadeDbIndexBuilderExtensions.GetPropertyName(propertyExpression),
68+
"property",
69+
null,
70+
null
71+
);
72+
73+
return this;
74+
}
75+
76+
public Bm25IndexBuilder<TEntity> HasField<TProperty>(
77+
Expression<Func<TEntity, TProperty>> propertyExpression,
78+
Tokenizer tokenizer
79+
)
80+
{
81+
AddField(
82+
ParadeDbIndexBuilderExtensions.GetPropertyName(propertyExpression),
83+
"property",
84+
tokenizer,
85+
null
86+
);
87+
88+
return this;
89+
}
90+
91+
public Bm25IndexBuilder<TEntity> HasField(string sql, Tokenizer tokenizer)
92+
{
93+
AddField(sql, "sql", tokenizer, null);
94+
95+
return this;
96+
}
97+
98+
public Bm25IndexBuilder<TEntity> HasField<TProperty>(
99+
Expression<Func<TEntity, TProperty>> propertyExpression,
100+
FieldAlias @alias
101+
)
102+
{
103+
AddField(
104+
ParadeDbIndexBuilderExtensions.GetPropertyName(propertyExpression),
105+
"property",
106+
null,
107+
@alias.name
108+
);
109+
110+
return this;
111+
}
112+
113+
public Bm25IndexBuilder<TEntity> HasField(string sql, FieldAlias @alias)
114+
{
115+
AddField(sql, "sql", null, @alias.name);
116+
117+
return this;
118+
}
119+
120+
public Bm25IndexBuilder<TEntity> HasFilter(string? sql)
121+
{
122+
_indexBuilder.HasFilter(sql);
123+
124+
return this;
125+
}
126+
127+
public Bm25IndexBuilder<TEntity> IsCreatedConcurrently(bool createdConcurrently = true)
128+
{
129+
_indexBuilder.IsCreatedConcurrently(createdConcurrently);
130+
131+
return this;
132+
}
133+
134+
public Bm25IndexBuilder<TEntity> HasSearchTokenizer(Tokenizer tokenizer)
135+
{
136+
_indexBuilder.HasAnnotation(
137+
ParadeDbAnnotationNames.Bm25SearchTokenizer,
138+
tokenizer.ToSearchString()
139+
);
140+
141+
return this;
142+
}
143+
144+
private void AddField(string field, string kind, Tokenizer? tokenizer, string? @alias)
145+
{
146+
var properties = GetAnnotation(ParadeDbAnnotationNames.Bm25FieldProperties);
147+
var kinds = GetAnnotation(ParadeDbAnnotationNames.Bm25FieldKinds);
148+
var tokenizers = GetAnnotation(ParadeDbAnnotationNames.Bm25FieldTokenizers);
149+
var aliases = GetAnnotation(ParadeDbAnnotationNames.Bm25FieldAliases);
150+
151+
_indexBuilder.HasAnnotation(
152+
ParadeDbAnnotationNames.Bm25FieldProperties,
153+
properties.Append(field).ToArray()
154+
);
155+
_indexBuilder.HasAnnotation(
156+
ParadeDbAnnotationNames.Bm25FieldKinds,
157+
kinds.Append(kind).ToArray()
158+
);
159+
_indexBuilder.HasAnnotation(
160+
ParadeDbAnnotationNames.Bm25FieldTokenizers,
161+
tokenizers.Append(tokenizer?.ToString() ?? "").ToArray()
162+
);
163+
164+
_indexBuilder.HasAnnotation(
165+
ParadeDbAnnotationNames.Bm25FieldAliases,
166+
aliases.Append(@alias is null ? "" : @alias.Replace("'", "''")).ToArray()
167+
);
168+
}
169+
170+
private string[] GetAnnotation(string name) =>
171+
(string[]?)_indexBuilder.Metadata.FindAnnotation(name)?.Value ?? [];
172+
}

‎src/Internal/Metadata/ParadeDbAnnotationNames.cs‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
namespace ParadeDB.EntityFrameworkCore.Internal.Metadata;
2+
3+
internal static class ParadeDbAnnotationNames
4+
{
5+
public const string Bm25KeyProperty = "ParadeDB:Bm25KeyProperty";
6+
public const string Bm25FieldProperties = "ParadeDB:Bm25FieldProperties";
7+
public const string Bm25FieldKinds = "ParadeDB:Bm25FieldKinds";
8+
public const string Bm25FieldTokenizers = "ParadeDB:Bm25FieldTokenizers";
9+
public const string Bm25FieldAliases = "ParadeDB:Bm25FieldAliases";
10+
public const string Bm25SearchTokenizer = "ParadeDB:Bm25SearchTokenizer";
11+
public const string Bm25KeyField = "ParadeDB:Bm25KeyField";
12+
public const string Bm25Fields = "ParadeDB:Bm25Fields";
13+
}

‎src/Internal/Metadata/ParadeDbAnnotationProvider.cs‎

Lines changed: 139 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,139 @@
1+
using Microsoft.EntityFrameworkCore;
2+
using Microsoft.EntityFrameworkCore.Infrastructure;
3+
using Microsoft.EntityFrameworkCore.Metadata;
4+
using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata.Internal;
5+
6+
namespace ParadeDB.EntityFrameworkCore.Internal.Metadata;
7+
8+
// NpgsqlAnnotationProvider is not part of the public API they explicitly note that it
9+
// is not subject to the same compatibility standards as the public API. So we may have to
10+
// rework things here to support new releases. I don't know that there is a way for us to
11+
// implement the indexing functionality in a way that only relies on the public API.
12+
internal sealed class ParadeDbAnnotationProvider : NpgsqlAnnotationProvider
13+
{
14+
public ParadeDbAnnotationProvider(RelationalAnnotationProviderDependencies dependencies)
15+
: base(dependencies) { }
16+
17+
public override IEnumerable<IAnnotation> For(ITableIndex index, bool designTime)
18+
{
19+
foreach (var annotation in base.For(index, designTime))
20+
{
21+
yield return annotation;
22+
}
23+
24+
var mappedIndex = index.MappedIndexes.FirstOrDefault(i =>
25+
i.FindAnnotation(ParadeDbAnnotationNames.Bm25KeyProperty)?.Value is string
26+
);
27+
if (
28+
mappedIndex?.FindAnnotation(ParadeDbAnnotationNames.Bm25KeyProperty)?.Value
29+
is not string keyPropertyName
30+
)
31+
{
32+
yield break;
33+
}
34+
35+
var fieldProperties = (string[])
36+
mappedIndex.FindAnnotation(ParadeDbAnnotationNames.Bm25FieldProperties)!.Value!;
37+
var fieldKinds = (string[])
38+
mappedIndex.FindAnnotation(ParadeDbAnnotationNames.Bm25FieldKinds)!.Value!;
39+
var fieldTokenizers = (string[])
40+
mappedIndex.FindAnnotation(ParadeDbAnnotationNames.Bm25FieldTokenizers)!.Value!;
41+
var fieldAliases = (string[])
42+
mappedIndex.FindAnnotation(ParadeDbAnnotationNames.Bm25FieldAliases)!.Value!;
43+
44+
if (
45+
fieldProperties.Length != fieldKinds.Length
46+
|| fieldProperties.Length != fieldTokenizers.Length
47+
|| fieldProperties.Length != fieldAliases.Length
48+
)
49+
{
50+
throw new InvalidOperationException(
51+
"A BM25 index must have one kind, tokenizer, and alias entry for each field."
52+
);
53+
}
54+
55+
var storeObject = StoreObjectIdentifier.Table(index.Table.Name, index.Table.Schema);
56+
57+
yield return new Annotation(
58+
ParadeDbAnnotationNames.Bm25KeyField,
59+
mappedIndex
60+
.DeclaringEntityType.FindProperty(keyPropertyName)!
61+
.GetColumnName(storeObject)!
62+
);
63+
64+
yield return new Annotation(
65+
ParadeDbAnnotationNames.Bm25Fields,
66+
fieldProperties
67+
.Select(
68+
(field, i) =>
69+
RenderField(
70+
mappedIndex.DeclaringEntityType,
71+
field,
72+
fieldKinds[i],
73+
storeObject,
74+
string.IsNullOrEmpty(fieldTokenizers[i]) ? null : fieldTokenizers[i],
75+
string.IsNullOrEmpty(fieldAliases[i]) ? null : fieldAliases[i]
76+
)
77+
)
78+
.ToArray()
79+
);
80+
81+
if (
82+
mappedIndex.FindAnnotation(ParadeDbAnnotationNames.Bm25SearchTokenizer)?.Value
83+
is string searchTokenizer
84+
)
85+
{
86+
yield return new Annotation(
87+
ParadeDbAnnotationNames.Bm25SearchTokenizer,
88+
searchTokenizer
89+
);
90+
}
91+
}
92+
93+
private static string RenderField(
94+
IReadOnlyEntityType entityType,
95+
string field,
96+
string kind,
97+
StoreObjectIdentifier storeObject,
98+
string? tokenizer,
99+
string? @alias
100+
)
101+
{
102+
var sql = kind switch
103+
{
104+
"property" => entityType.FindProperty(field)!.GetColumnName(storeObject)!,
105+
"sql" => field,
106+
_ => throw new InvalidOperationException($"Unknown field kind '{kind}'"),
107+
};
108+
109+
if (tokenizer is null && @alias is null)
110+
{
111+
return sql;
112+
}
113+
114+
if (tokenizer is not null && @alias is not null)
115+
{
116+
throw new InvalidOperationException(
117+
$"A field may not have a tokenizer and alias at the same time {tokenizer}, {@alias}"
118+
);
119+
}
120+
121+
if (tokenizer is not null)
122+
{
123+
return kind == "property" ? $"({sql}::{tokenizer})" : $"(({sql})::{tokenizer})";
124+
}
125+
126+
return kind == "property"
127+
? $"({sql}::pdb.alias('{@alias}'))"
128+
: $"(({sql})::pdb.alias('{@alias}'))";
129+
}
130+
131+
private static string GetColumnName(
132+
IReadOnlyEntityType entityType,
133+
string propertyName,
134+
StoreObjectIdentifier storeObject
135+
)
136+
{
137+
return entityType.FindProperty(propertyName)!.GetColumnName(storeObject)!;
138+
}
139+
}

0 commit comments

Comments
 (0)