MB-62985: Forcing cosine similarity for backing indexes within bq (#2282)

- Forcing cosine similarity for binary indexes during indexing and query
times
- Added relevant test cases

Author: Likith101

document/field_vector.go

@@ -114,6 +114,13 @@ func NewVectorFieldWithIndexingOptions(name string, arrayPositions []uint64,
 	// skip freq/norms for vector field
 	options |= index.SkipFreqNorm
 
+	// bivf-sq8 indexes only supports hamming distance for the primary
+	// binary index. Similarity here is used for the backing flat index,
+	// which is set to cosine similarity for recall reasons
+	if index.OptimizationRequiresBinaryIndex(vectorIndexOptimizedFor) {
+		similarity = index.CosineSimilarity
+	}
+
 	return &VectorField{
 		name:                    name,
 		dims:                    dims,

go.mod

@@ -5,9 +5,9 @@ go 1.24.0
 require (
 	github.com/RoaringBitmap/roaring/v2 v2.14.4
 	github.com/bits-and-blooms/bitset v1.24.2
-	github.com/blevesearch/bleve_index_api v1.3.2
+	github.com/blevesearch/bleve_index_api v1.3.3-0.20260311083042-62a5343ee2d1
 	github.com/blevesearch/geo v0.2.5
-	github.com/blevesearch/go-faiss v1.0.27
+	github.com/blevesearch/go-faiss v1.0.28-0.20260311083224-baa72a4cd87f
 	github.com/blevesearch/go-metrics v0.0.0-20201227073835-cf1acfcdf475
 	github.com/blevesearch/go-porterstemmer v1.0.3
 	github.com/blevesearch/goleveldb v1.0.1
@@ -25,7 +25,7 @@ require (
 	github.com/blevesearch/zapx/v14 v14.4.3
 	github.com/blevesearch/zapx/v15 v15.4.3
 	github.com/blevesearch/zapx/v16 v16.3.1
-	github.com/blevesearch/zapx/v17 v17.0.2
+	github.com/blevesearch/zapx/v17 v17.0.3-0.20260311100439-7e7434b4f844
 	github.com/couchbase/moss v0.2.0
 	github.com/spf13/cobra v1.10.2
 	go.etcd.io/bbolt v1.4.0

go.sum

@@ -2,12 +2,12 @@ github.com/RoaringBitmap/roaring/v2 v2.14.4 h1:4aKySrrg9G/5oRtJ3TrZLObVqxgQ9f1zn
 github.com/RoaringBitmap/roaring/v2 v2.14.4/go.mod h1:oMvV6omPWr+2ifRdeZvVJyaz+aoEUopyv5iH0u/+wbY=
 github.com/bits-and-blooms/bitset v1.24.2 h1:M7/NzVbsytmtfHbumG+K2bremQPMJuqv1JD3vOaFxp0=
 github.com/bits-and-blooms/bitset v1.24.2/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
-github.com/blevesearch/bleve_index_api v1.3.2 h1:y4VLXBF7nQR01CvF+QzmCJKMpVPCLp1CJ5FsRSZXzRE=
-github.com/blevesearch/bleve_index_api v1.3.2/go.mod h1:xvd48t5XMeeioWQ5/jZvgLrV98flT2rdvEJ3l/ki4Ko=
+github.com/blevesearch/bleve_index_api v1.3.3-0.20260311083042-62a5343ee2d1 h1:k2Q/e830m+fHx/4djya4UFef/jCV53sr0s97n9ergEc=
+github.com/blevesearch/bleve_index_api v1.3.3-0.20260311083042-62a5343ee2d1/go.mod h1:xvd48t5XMeeioWQ5/jZvgLrV98flT2rdvEJ3l/ki4Ko=
 github.com/blevesearch/geo v0.2.5 h1:yJg9FX1oRwLnjXSXF+ECHfXFTF4diF02Ca/qUGVjJhE=
 github.com/blevesearch/geo v0.2.5/go.mod h1:Jhq7WE2K6mJTx1xS44M2pUO6Io+wjCSHh1+co3YOgH4=
-github.com/blevesearch/go-faiss v1.0.27 h1:7cBImYDDQ82WJd5RUZ1ie6zXztCsC73W94ZzwOjkatk=
-github.com/blevesearch/go-faiss v1.0.27/go.mod h1:OMGQwOaRRYxrmeNdMrXJPvVx8gBnvE5RYrr0BahNnkk=
+github.com/blevesearch/go-faiss v1.0.28-0.20260311083224-baa72a4cd87f h1:BPOdM0yGnr74P3NelllUa8BbvKxqY5UfCUunggoGwoM=
+github.com/blevesearch/go-faiss v1.0.28-0.20260311083224-baa72a4cd87f/go.mod h1:OMGQwOaRRYxrmeNdMrXJPvVx8gBnvE5RYrr0BahNnkk=
 github.com/blevesearch/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:kDy+zgJFJJoJYBvdfBSiZYBbdsUL0XcjHYWezpQBGPA=
 github.com/blevesearch/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:9eJDeqxJ3E7WnLebQUlPD7ZjSce7AnDb9vjGmMCbD0A=
 github.com/blevesearch/go-porterstemmer v1.0.3 h1:GtmsqID0aZdCSNiY8SkuPJ12pD4jI+DdXTAn4YRcHCo=
@@ -45,8 +45,8 @@ github.com/blevesearch/zapx/v15 v15.4.3 h1:iJiMJOHrz216jyO6lS0m9RTCEkprUnzvqAI2l
 github.com/blevesearch/zapx/v15 v15.4.3/go.mod h1:1pssev/59FsuWcgSnTa0OeEpOzmhtmr/0/11H0Z8+Nw=
 github.com/blevesearch/zapx/v16 v16.3.1 h1:ERxZUSC9UcuKggCQ6b3y4sTkyL4WnGOWuopzglR874g=
 github.com/blevesearch/zapx/v16 v16.3.1/go.mod h1:zCFjv7McXWm1C8rROL+3mUoD5WYe2RKsZP3ufqcYpLY=
-github.com/blevesearch/zapx/v17 v17.0.2 h1:u0VpuBIKv5bDvtXMOEOwxsqZe30VFrDaE/QKG831CLQ=
-github.com/blevesearch/zapx/v17 v17.0.2/go.mod h1:70Z2efizAb00jpeLJA7r8DuqWzEtUdcLcWqaHIaOols=
+github.com/blevesearch/zapx/v17 v17.0.3-0.20260311100439-7e7434b4f844 h1:IMv2az9s66lo/7+RtsJek+Lanu9cnwU6a+YDwoh9m+s=
+github.com/blevesearch/zapx/v17 v17.0.3-0.20260311100439-7e7434b4f844/go.mod h1:/TdOJs9QM/V/j6OSr14BHdAGO5cnLpa9qhX6Np9A/Ts=
 github.com/couchbase/ghistogram v0.1.0 h1:b95QcQTCzjTUocDXp/uMgSNQi8oj1tGwnJ4bODWZnps=
 github.com/couchbase/ghistogram v0.1.0/go.mod h1:s1Jhy76zqfEecpNWJfWUiKZookAFaiGOEoyzgHt9i7k=
 github.com/couchbase/moss v0.2.0 h1:VCYrMzFwEryyhRSeI+/b3tRBSeTpi/8gn5Kf6dxqn+o=