-
-
Notifications
You must be signed in to change notification settings - Fork 1
feat: native vector search support #90
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: paradedb-am
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,7 +1,3 @@ | ||
| <Project> | ||
| <Import Project="..\Directory.Packages.props" /> | ||
|
|
||
| <ItemGroup> | ||
| <PackageVersion Include="Pgvector.EntityFrameworkCore" Version="0.3.0" /> | ||
| </ItemGroup> | ||
| </Project> |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,9 +1,8 @@ | ||
| using Pgvector; | ||
| using Shared; | ||
|
|
||
| namespace HybridRrf.Data; | ||
|
|
||
| public class MockItemWithEmbedding : MockItem | ||
| { | ||
| public Vector? Embedding { get; set; } | ||
| public float[]? Embedding { get; set; } | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,39 @@ | ||
| using HybridRrf.Data; | ||
| using Microsoft.EntityFrameworkCore; | ||
|
|
||
| namespace HybridRrf; | ||
|
|
||
| public static class EmbeddingLoader | ||
| { | ||
| public static async Task LoadAsync(AppDbContext db) | ||
| { | ||
| var csvPath = Path.Combine( | ||
| AppContext.BaseDirectory, | ||
| "HybridRrf", | ||
| "mock_items_embeddings.csv" | ||
| ); | ||
| var lines = await File.ReadAllLinesAsync(csvPath); | ||
|
|
||
| var embeddings = new Dictionary<int, float[]>(); | ||
|
|
||
| for (var i = 1; i < lines.Length; i++) | ||
| { | ||
| var parts = lines[i].Split(',', 3); | ||
| embeddings[int.Parse(parts[0])] = parts[2] | ||
| .Trim('"', '[', ']') | ||
| .Split(',', StringSplitOptions.TrimEntries) | ||
| .Select(float.Parse) | ||
| .ToArray(); | ||
| } | ||
|
|
||
| var ids = embeddings.Keys.ToArray(); | ||
| var items = await db.MockItems.Where(x => ids.Contains(x.Id)).ToListAsync(); | ||
| foreach (var item in items) | ||
| { | ||
| item.Embedding = embeddings[item.Id]; | ||
| } | ||
|
|
||
| await db.SaveChangesAsync(); | ||
| Console.WriteLine($"Loaded {items.Count} embeddings"); | ||
| } | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -46,6 +46,9 @@ await db.Database.ExecuteSqlRawAsync( | |
| "CALL paradedb.create_bm25_test_table(schema_name => 'public', table_name => 'mock_items')" | ||
| ); | ||
| await db.Database.ExecuteSqlRawAsync("DROP INDEX IF EXISTS mock_items_bm25_idx"); | ||
| // The VectorSearch example may have left its own index behind, and newer | ||
| // pg_search versions allow only one ParadeDB index per table | ||
| await db.Database.ExecuteSqlRawAsync("DROP INDEX IF EXISTS mock_items_vector_idx"); | ||
|
Comment on lines
48
to
+51
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. We should update all examples to use one index name between them. Maybe |
||
| await db.Database.ExecuteSqlRawAsync( | ||
| """ | ||
| CREATE INDEX mock_items_bm25_idx ON mock_items | ||
|
|
@@ -102,14 +105,18 @@ public static async Task SetupHybridAsync(DbContext db) | |
| { | ||
| await SetupMockItemsAsync(db); | ||
| await db.Database.ExecuteSqlRawAsync("CREATE EXTENSION IF NOT EXISTS vector"); | ||
| // Newer pg_search versions create mock_items with an embedding column of a | ||
| // different dimension; the examples use 384-dimensional embeddings | ||
| await db.Database.ExecuteSqlRawAsync( | ||
| """ | ||
| DO $$ | ||
| BEGIN | ||
| IF NOT EXISTS ( | ||
| SELECT 1 FROM information_schema.columns | ||
| WHERE table_name = 'mock_items' AND column_name = 'embedding' | ||
| ) THEN | ||
| IF ( | ||
| SELECT atttypmod FROM pg_attribute | ||
| WHERE attrelid = 'mock_items'::regclass | ||
| AND attname = 'embedding' AND NOT attisdropped | ||
| ) IS DISTINCT FROM 384 THEN | ||
| ALTER TABLE mock_items DROP COLUMN IF EXISTS embedding; | ||
| ALTER TABLE mock_items ADD COLUMN embedding vector(384); | ||
| END IF; | ||
| END $$; | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,65 @@ | ||
| using HybridRrf; | ||
| using HybridRrf.Data; | ||
| using Microsoft.EntityFrameworkCore; | ||
| using ParadeDB.EntityFrameworkCore.Extensions; | ||
| using Shared; | ||
|
|
||
| var options = new DbContextOptionsBuilder<AppDbContext>() | ||
| .UseNpgsql(ExampleSetup.ConnectionString, o => o.UseParadeDb()) | ||
| .UseSnakeCaseNamingConvention() | ||
| .Options; | ||
|
|
||
| await using var dbContext = new AppDbContext(options); | ||
|
|
||
| Console.WriteLine(new string('=', 70)); | ||
| Console.WriteLine("Vector Search"); | ||
| Console.WriteLine(new string('=', 70)); | ||
| Console.WriteLine("\nTop-K nearest-neighbor search over a pgvector column"); | ||
|
|
||
| await ExampleSetup.SetupHybridAsync(dbContext); | ||
| await EmbeddingLoader.LoadAsync(dbContext); | ||
|
|
||
| // Newer pg_search versions allow only one ParadeDB index per table | ||
| await dbContext.Database.ExecuteSqlRawAsync("DROP INDEX IF EXISTS mock_items_bm25_idx"); | ||
| await dbContext.Database.ExecuteSqlRawAsync("DROP INDEX IF EXISTS mock_items_vector_idx"); | ||
| await dbContext.Database.ExecuteSqlRawAsync( | ||
| """ | ||
| CREATE INDEX mock_items_vector_idx ON mock_items | ||
| USING paradedb (id, embedding vector_cosine_ops) | ||
| WITH (key_field='id'); | ||
| """ | ||
| ); | ||
| Console.WriteLine("Created a paradedb index over the embedding column (cosine metric)"); | ||
|
|
||
| await Demo(dbContext, "running shoes"); | ||
| await Demo(dbContext, "wireless earbuds"); | ||
| return; | ||
|
|
||
| static async Task Demo(AppDbContext db, string query) | ||
| { | ||
| var queryEmbedding = QueryEmbeddings.Values[query]; | ||
|
|
||
| // The @@@ predicate (EF.Functions.All) and the LIMIT are required for the | ||
| // ParadeDB index to serve the query; the ORDER BY metric must match the | ||
| // index opclass (cosine here) | ||
| var results = await db | ||
| .MockItems.Where(x => EF.Functions.All(x.Id)) | ||
| .Select(x => new | ||
| { | ||
| x.Description, | ||
| Distance = EF.Functions.CosineDistance(x.Embedding, queryEmbedding), | ||
| }) | ||
| .OrderBy(x => x.Distance) | ||
| .Take(5) | ||
| .ToListAsync(); | ||
|
|
||
| Console.WriteLine($"\n{new string('=', 70)}"); | ||
| Console.WriteLine($"Query: '{query}'"); | ||
| Console.WriteLine(new string('=', 70)); | ||
|
|
||
| for (var i = 0; i < results.Count; i++) | ||
| { | ||
| var desc = results[i].Description[..Math.Min(60, results[i].Description.Length)]; | ||
| Console.WriteLine($" {i + 1}. {desc, -60} (distance: {results[i].Distance:F4})"); | ||
| } | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -13,6 +13,7 @@ examples=( | |
| Autocomplete | ||
| MoreLikeThis | ||
| HybridRrf | ||
| VectorSearch | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. For the other ORMs I see you put VectorSearch below Rag. I think it looks better above, but that's a nit |
||
| Rag | ||
| ) | ||
|
|
||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Should we add
pgvectoras a requirement in these tables?