Skip to content

Commit e92dbd3

Browse files
Significantly speed up normalization
1 parent d21e011 commit e92dbd3

1 file changed

Lines changed: 6 additions & 4 deletions

File tree

Sources/MagnitudeDB/MagnitudeDatabase.swift

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -229,17 +229,19 @@ extension MagnitudeDatabase {
229229
@discardableResult
230230
public func normalizeDatabase() throws -> Int {
231231
let documentsTable = Table("documents")
232+
let documentEmbedding = Expression<[Float]>("embedding")
233+
let documentID = Expression<Int>("id")
232234
var numberNormalized = 0
233235

234236
for raw in try db.prepare(documentsTable) {
235-
var document: Document = try raw.decode()
236-
guard document.embedding.contains(where: { $0 > 1 }) else {
237+
var embedding = try raw.get(documentEmbedding)
238+
guard embedding.contains(where: { $0 > 1 }) else {
237239
continue // Already normalized
238240
}
239241

240-
faiss_fvec_renorm_L2(vectorDimensions, 1, &document.embedding)
242+
faiss_fvec_renorm_L2(vectorDimensions, 1, &embedding)
241243

242-
_ = try documentsTable.update(document)
244+
_ = try documentsTable.where(documentID == raw.get(documentID)).update(documentEmbedding <- embedding)
243245
numberNormalized += 1
244246
}
245247

0 commit comments

Comments
 (0)