Skip to content

Commit ca7d4ab

Browse files
CascadingRadiumCopilotabhinavdangeti
authored
MB-27666: Hierarchy Search (#2224)
Add support for nested fields in indexing and querying - Parse and index nested JSON objects - Enable queries on nested fields - Preserve hierarchical relationships in index Requires: - blevesearch/bleve_index_api#70 - https://github.com/blevesearch/bleve_index_api/releases/tag/v1.3.0 - blevesearch/scorch_segment_api#63 - https://github.com/blevesearch/scorch_segment_api/releases/tag/v2.4.0 - blevesearch/zapx#339, blevesearch/zapx#365 - https://github.com/blevesearch/zapx/releases/tag/v17.0.0 - https://github.com/blevesearch/zapx/releases/tag/v16.3.0 Resolves: - #15 - #637 - #1297 --------- Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Co-authored-by: Abhinav Dangeti <abhinav@couchbase.com>
1 parent 32d9882 commit ca7d4ab

41 files changed

Lines changed: 3133 additions & 161 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

README.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ A modern indexing + search library in GO
2424
* [geo spatial search](https://github.com/blevesearch/bleve/blob/master/geo/README.md)
2525
* approximate k-nearest neighbors via [vector search](https://github.com/blevesearch/bleve/blob/master/docs/vectors.md)
2626
* [synonym search](https://github.com/blevesearch/bleve/blob/master/docs/synonyms.md)
27+
* [hierarchical nested search](https://github.com/blevesearch/bleve/blob/master/docs/hierarchy.md)
2728
* [tf-idf](https://github.com/blevesearch/bleve/blob/master/docs/scoring.md#tf-idf) / [bm25](https://github.com/blevesearch/bleve/blob/master/docs/scoring.md#bm25) scoring models
2829
* Hybrid search: exact + semantic
2930
* Supports [RRF (Reciprocal Rank Fusion) and RSF (Relative Score Fusion)](docs/score_fusion.md)

docs/hierarchy.md

Lines changed: 376 additions & 0 deletions
Large diffs are not rendered by default.

docs/vectors.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
| `v2.5.0`, `v2.5.1` | [blevesearch/faiss@352484e](https://github.com/blevesearch/faiss/tree/352484e0fc9d1f8f46737841efe5f26e0f383f71) (modified v1.10.0) |
2121
| `v2.5.2`, `v2.5.3`, `v2.5.4` | [blevesearch/faiss@b3d4e00](https://github.com/blevesearch/faiss/tree/b3d4e00a69425b95e0b283da7801efc9f66b580d) (modified v1.11.0) |
2222
| `v2.5.5`, `v2.5.6`, `v2.5.7` | [blevesearch/faiss@8a59a0c](https://github.com/blevesearch/faiss/tree/8a59a0c552fa2d14fa871f6b6bc793de1d277f5e) (modified v1.12.0) |
23+
| `v2.6.0` | [blevesearch/faiss@608356b](https://github.com/blevesearch/faiss/tree/608356b7c9630e891ff87cc49cc7bb460c3870d3) (modified v1.13.1) |
2324

2425
## Supported
2526

document/document.go

Lines changed: 35 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import (
1818
"fmt"
1919
"reflect"
2020

21+
"github.com/blevesearch/bleve/v2/search"
2122
"github.com/blevesearch/bleve/v2/size"
2223
index "github.com/blevesearch/bleve_index_api"
2324
)
@@ -30,8 +31,9 @@ func init() {
3031
}
3132

3233
type Document struct {
33-
id string `json:"id"`
34-
Fields []Field `json:"fields"`
34+
id string
35+
Fields []Field `json:"fields"`
36+
NestedDocuments []*Document `json:"nested_documents"`
3537
CompositeFields []*CompositeField
3638
StoredFieldsSize uint64
3739
indexed bool
@@ -157,3 +159,34 @@ func (d *Document) SetIndexed() {
157159
func (d *Document) Indexed() bool {
158160
return d.indexed
159161
}
162+
163+
func (d *Document) AddNestedDocument(doc *Document) {
164+
d.NestedDocuments = append(d.NestedDocuments, doc)
165+
}
166+
167+
func (d *Document) NestedFields() search.FieldSet {
168+
if len(d.NestedDocuments) == 0 {
169+
return nil
170+
}
171+
fieldSet := search.NewFieldSet()
172+
var collectFields func(index.Document)
173+
collectFields = func(doc index.Document) {
174+
// Add all field names from this nested document
175+
doc.VisitFields(func(field index.Field) {
176+
fieldSet.AddField(field.Name())
177+
})
178+
// Recursively collect from this document's nested documents
179+
if nd, ok := doc.(index.NestedDocument); ok {
180+
nd.VisitNestedDocuments(collectFields)
181+
}
182+
}
183+
// Start collection from nested documents only (not root document)
184+
d.VisitNestedDocuments(collectFields)
185+
return fieldSet
186+
}
187+
188+
func (d *Document) VisitNestedDocuments(visitor func(doc index.Document)) {
189+
for _, doc := range d.NestedDocuments {
190+
visitor(doc)
191+
}
192+
}

go.mod

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,14 +5,14 @@ go 1.24
55
require (
66
github.com/RoaringBitmap/roaring/v2 v2.4.5
77
github.com/bits-and-blooms/bitset v1.22.0
8-
github.com/blevesearch/bleve_index_api v1.2.12-0.20260109154621-f19a6d6af728
8+
github.com/blevesearch/bleve_index_api v1.3.0
99
github.com/blevesearch/geo v0.2.4
1010
github.com/blevesearch/go-faiss v1.0.27
1111
github.com/blevesearch/go-metrics v0.0.0-20201227073835-cf1acfcdf475
1212
github.com/blevesearch/go-porterstemmer v1.0.3
1313
github.com/blevesearch/goleveldb v1.0.1
1414
github.com/blevesearch/gtreap v0.1.1
15-
github.com/blevesearch/scorch_segment_api/v2 v2.3.14-0.20260109154938-b56b54c737df
15+
github.com/blevesearch/scorch_segment_api/v2 v2.4.0
1616
github.com/blevesearch/segment v0.9.1
1717
github.com/blevesearch/snowball v0.6.1
1818
github.com/blevesearch/snowballstem v0.9.0
@@ -24,8 +24,8 @@ require (
2424
github.com/blevesearch/zapx/v13 v13.4.2
2525
github.com/blevesearch/zapx/v14 v14.4.2
2626
github.com/blevesearch/zapx/v15 v15.4.2
27-
github.com/blevesearch/zapx/v16 v16.2.8
28-
github.com/blevesearch/zapx/v17 v17.0.0-20260112205515-7d8cac80436c
27+
github.com/blevesearch/zapx/v16 v16.3.0
28+
github.com/blevesearch/zapx/v17 v17.0.0
2929
github.com/couchbase/moss v0.2.0
3030
github.com/spf13/cobra v1.8.1
3131
go.etcd.io/bbolt v1.4.0

go.sum

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,8 @@ github.com/RoaringBitmap/roaring/v2 v2.4.5/go.mod h1:FiJcsfkGje/nZBZgCu0ZxCPOKD/
33
github.com/bits-and-blooms/bitset v1.12.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
44
github.com/bits-and-blooms/bitset v1.22.0 h1:Tquv9S8+SGaS3EhyA+up3FXzmkhxPGjQQCkcs2uw7w4=
55
github.com/bits-and-blooms/bitset v1.22.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
6-
github.com/blevesearch/bleve_index_api v1.2.12-0.20260109154621-f19a6d6af728 h1:qFnvr+SqVOCbhMl5sVynhuwVkv1yrc7Vhrn8lVdw1nU=
7-
github.com/blevesearch/bleve_index_api v1.2.12-0.20260109154621-f19a6d6af728/go.mod h1:xvd48t5XMeeioWQ5/jZvgLrV98flT2rdvEJ3l/ki4Ko=
6+
github.com/blevesearch/bleve_index_api v1.3.0 h1:DsMpWVjFNlBw9/6pyWf59XoqcAkhHj3H0UWiQsavb6E=
7+
github.com/blevesearch/bleve_index_api v1.3.0/go.mod h1:xvd48t5XMeeioWQ5/jZvgLrV98flT2rdvEJ3l/ki4Ko=
88
github.com/blevesearch/geo v0.2.4 h1:ECIGQhw+QALCZaDcogRTNSJYQXRtC8/m8IKiA706cqk=
99
github.com/blevesearch/geo v0.2.4/go.mod h1:K56Q33AzXt2YExVHGObtmRSFYZKYGv0JEN5mdacJJR8=
1010
github.com/blevesearch/go-faiss v1.0.27 h1:7cBImYDDQ82WJd5RUZ1ie6zXztCsC73W94ZzwOjkatk=
@@ -20,8 +20,8 @@ github.com/blevesearch/gtreap v0.1.1/go.mod h1:QaQyDRAT51sotthUWAH4Sj08awFSSWzgY
2020
github.com/blevesearch/mmap-go v1.0.2/go.mod h1:ol2qBqYaOUsGdm7aRMRrYGgPvnwLe6Y+7LMvAB5IbSA=
2121
github.com/blevesearch/mmap-go v1.0.4 h1:OVhDhT5B/M1HNPpYPBKIEJaD0F3Si+CrEKULGCDPWmc=
2222
github.com/blevesearch/mmap-go v1.0.4/go.mod h1:EWmEAOmdAS9z/pi/+Toxu99DnsbhG1TIxUoRmJw/pSs=
23-
github.com/blevesearch/scorch_segment_api/v2 v2.3.14-0.20260109154938-b56b54c737df h1:gBuVkzZLUpGJGnCBRgY0ruZVjppD7WaQLeHZei7QQnU=
24-
github.com/blevesearch/scorch_segment_api/v2 v2.3.14-0.20260109154938-b56b54c737df/go.mod h1:f8fXitmMpzgNziIMqUlpTrfPxVVDN8at9k7POEohvJU=
23+
github.com/blevesearch/scorch_segment_api/v2 v2.4.0 h1:OtipwURRzZv6UFmHQnbEqOY90eotINQ2TtSSpWfYuWU=
24+
github.com/blevesearch/scorch_segment_api/v2 v2.4.0/go.mod h1:JalWE/eyEgISwhqtKXoaHMKf5t+F4kXiYrgg0ds3ylw=
2525
github.com/blevesearch/segment v0.9.1 h1:+dThDy+Lvgj5JMxhmOVlgFfkUtZV2kw49xax4+jTfSU=
2626
github.com/blevesearch/segment v0.9.1/go.mod h1:zN21iLm7+GnBHWTao9I+Au/7MBiL8pPFtJBJTsk6kQw=
2727
github.com/blevesearch/snowball v0.6.1 h1:cDYjn/NCH+wwt2UdehaLpr2e4BwLIjN4V/TdLsL+B5A=
@@ -44,10 +44,10 @@ github.com/blevesearch/zapx/v14 v14.4.2 h1:2SGHakVKd+TrtEqpfeq8X+So5PShQ5nW6GNxT
4444
github.com/blevesearch/zapx/v14 v14.4.2/go.mod h1:rz0XNb/OZSMjNorufDGSpFpjoFKhXmppH9Hi7a877D8=
4545
github.com/blevesearch/zapx/v15 v15.4.2 h1:sWxpDE0QQOTjyxYbAVjt3+0ieu8NCE0fDRaFxEsp31k=
4646
github.com/blevesearch/zapx/v15 v15.4.2/go.mod h1:1pssev/59FsuWcgSnTa0OeEpOzmhtmr/0/11H0Z8+Nw=
47-
github.com/blevesearch/zapx/v16 v16.2.8 h1:SlnzF0YGtSlrsOE3oE7EgEX6BIepGpeqxs1IjMbHLQI=
48-
github.com/blevesearch/zapx/v16 v16.2.8/go.mod h1:murSoCJPCk25MqURrcJaBQ1RekuqSCSfMjXH4rHyA14=
49-
github.com/blevesearch/zapx/v17 v17.0.0-20260112205515-7d8cac80436c h1:OfYh0noLbJmt6k2tqYlnSU3zMZEJbFfbSClSGG59A/M=
50-
github.com/blevesearch/zapx/v17 v17.0.0-20260112205515-7d8cac80436c/go.mod h1:ybWwo00MGrNJuFDnl9smEBVUCZmNANf0+E/QVBmfBTs=
47+
github.com/blevesearch/zapx/v16 v16.3.0 h1:hF6VlN15E9CB40RMPyqOIhlDw1OOo9RItumhKMQktxw=
48+
github.com/blevesearch/zapx/v16 v16.3.0/go.mod h1:zCFjv7McXWm1C8rROL+3mUoD5WYe2RKsZP3ufqcYpLY=
49+
github.com/blevesearch/zapx/v17 v17.0.0 h1:srLJFkv5ghz1Z8iVz5uoOK89G2NvI4KdMG7aF3Cx7rE=
50+
github.com/blevesearch/zapx/v17 v17.0.0/go.mod h1:/pi9Gq7byQcduhNB6Vk08+ZXGVGPjZoNc5QnQY8lkOo=
5151
github.com/couchbase/ghistogram v0.1.0 h1:b95QcQTCzjTUocDXp/uMgSNQi8oj1tGwnJ4bODWZnps=
5252
github.com/couchbase/ghistogram v0.1.0/go.mod h1:s1Jhy76zqfEecpNWJfWUiKZookAFaiGOEoyzgHt9i7k=
5353
github.com/couchbase/moss v0.2.0 h1:VCYrMzFwEryyhRSeI+/b3tRBSeTpi/8gn5Kf6dxqn+o=

index/scorch/introducer.go

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -170,6 +170,11 @@ func (s *Scorch) introduceSegment(next *segmentIntroduction) error {
170170
newss.deleted = nil
171171
}
172172

173+
// update the deleted bitmap to include any nested/sub-documents as well
174+
// if the segment supports that
175+
if ns, ok := newss.segment.(segment.NestedSegment); ok {
176+
newss.deleted = ns.AddNestedDocuments(newss.deleted)
177+
}
173178
// check for live size before copying
174179
if newss.LiveSize() > 0 {
175180
newSnapshot.segment = append(newSnapshot.segment, newss)

index/scorch/scorch.go

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -799,6 +799,12 @@ func analyze(d index.Document, fn customAnalyzerPluginInitFunc) {
799799
}
800800
}
801801
})
802+
if nd, ok := d.(index.NestedDocument); ok {
803+
nd.VisitNestedDocuments(func(doc index.Document) {
804+
doc.AddIDField()
805+
analyze(doc, fn)
806+
})
807+
}
802808
}
803809

804810
func (s *Scorch) AddEligibleForRemoval(epoch uint64) {

index/scorch/snapshot_index.go

Lines changed: 29 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@ package scorch
1717
import (
1818
"container/heap"
1919
"context"
20-
"encoding/binary"
2120
"fmt"
2221
"os"
2322
"path/filepath"
@@ -42,9 +41,8 @@ type asynchSegmentResult struct {
4241
dict segment.TermDictionary
4342
dictItr segment.DictionaryIterator
4443

45-
cardinality int
46-
index int
47-
docs *roaring.Bitmap
44+
index int
45+
docs *roaring.Bitmap
4846

4947
thesItr segment.ThesaurusIterator
5048

@@ -59,11 +57,11 @@ func init() {
5957
var err error
6058
lb1, err = lev.NewLevenshteinAutomatonBuilder(1, true)
6159
if err != nil {
62-
panic(fmt.Errorf("Levenshtein automaton ed1 builder err: %v", err))
60+
panic(fmt.Errorf("levenshtein automaton ed1 builder err: %v", err))
6361
}
6462
lb2, err = lev.NewLevenshteinAutomatonBuilder(2, true)
6563
if err != nil {
66-
panic(fmt.Errorf("Levenshtein automaton ed2 builder err: %v", err))
64+
panic(fmt.Errorf("levenshtein automaton ed2 builder err: %v", err))
6765
}
6866
}
6967

@@ -474,7 +472,7 @@ func (is *IndexSnapshot) GetInternal(key []byte) ([]byte, error) {
474472
func (is *IndexSnapshot) DocCount() (uint64, error) {
475473
var rv uint64
476474
for _, segment := range is.segment {
477-
rv += segment.Count()
475+
rv += segment.CountRoot()
478476
}
479477
return rv, nil
480478
}
@@ -501,7 +499,7 @@ func (is *IndexSnapshot) Document(id string) (rv index.Document, err error) {
501499
return nil, nil
502500
}
503501

504-
docNum, err := docInternalToNumber(next.ID)
502+
docNum, err := next.ID.Value()
505503
if err != nil {
506504
return nil, err
507505
}
@@ -571,7 +569,7 @@ func (is *IndexSnapshot) segmentIndexAndLocalDocNumFromGlobal(docNum uint64) (in
571569
}
572570

573571
func (is *IndexSnapshot) ExternalID(id index.IndexInternalID) (string, error) {
574-
docNum, err := docInternalToNumber(id)
572+
docNum, err := id.Value()
575573
if err != nil {
576574
return "", err
577575
}
@@ -589,7 +587,7 @@ func (is *IndexSnapshot) ExternalID(id index.IndexInternalID) (string, error) {
589587
}
590588

591589
func (is *IndexSnapshot) segmentIndexAndLocalDocNum(id index.IndexInternalID) (int, uint64, error) {
592-
docNum, err := docInternalToNumber(id)
590+
docNum, err := id.Value()
593591
if err != nil {
594592
return 0, 0, err
595593
}
@@ -778,25 +776,6 @@ func (is *IndexSnapshot) recycleTermFieldReader(tfr *IndexSnapshotTermFieldReade
778776
is.m2.Unlock()
779777
}
780778

781-
func docNumberToBytes(buf []byte, in uint64) []byte {
782-
if len(buf) != 8 {
783-
if cap(buf) >= 8 {
784-
buf = buf[0:8]
785-
} else {
786-
buf = make([]byte, 8)
787-
}
788-
}
789-
binary.BigEndian.PutUint64(buf, in)
790-
return buf
791-
}
792-
793-
func docInternalToNumber(in index.IndexInternalID) (uint64, error) {
794-
if len(in) != 8 {
795-
return 0, fmt.Errorf("wrong len for IndexInternalID: %q", in)
796-
}
797-
return binary.BigEndian.Uint64(in), nil
798-
}
799-
800779
func (is *IndexSnapshot) documentVisitFieldTermsOnSegment(
801780
segmentIndex int, localDocNum uint64, fields []string, cFields []string,
802781
visitor index.DocValueVisitor, dvs segment.DocVisitState) (
@@ -899,7 +878,7 @@ func (dvr *DocValueReader) BytesRead() uint64 {
899878
func (dvr *DocValueReader) VisitDocValues(id index.IndexInternalID,
900879
visitor index.DocValueVisitor,
901880
) (err error) {
902-
docNum, err := docInternalToNumber(id)
881+
docNum, err := id.Value()
903882
if err != nil {
904883
return err
905884
}
@@ -1299,3 +1278,23 @@ func (is *IndexSnapshot) TermFrequencies(field string, limit int, descending boo
12991278

13001279
return termFreqs[:limit], nil
13011280
}
1281+
1282+
// Ancestors returns the ancestor IDs for the given document ID. The prealloc
1283+
// slice can be provided to avoid allocations downstream, and MUST be empty.
1284+
func (i *IndexSnapshot) Ancestors(ID index.IndexInternalID, prealloc []index.AncestorID) ([]index.AncestorID, error) {
1285+
// get segment and local doc num for the ID
1286+
seg, ldoc, err := i.segmentIndexAndLocalDocNum(ID)
1287+
if err != nil {
1288+
return nil, err
1289+
}
1290+
// get ancestors from the segment
1291+
prealloc = i.segment[seg].Ancestors(ldoc, prealloc)
1292+
// get global offset for the segment (correcting factor for multi-segment indexes)
1293+
globalOffset := i.offsets[seg]
1294+
// adjust ancestors to global doc numbers, not local to segment
1295+
for idx := range prealloc {
1296+
prealloc[idx] = prealloc[idx].Add(globalOffset)
1297+
}
1298+
// return adjusted ancestors
1299+
return prealloc, nil
1300+
}

index/scorch/snapshot_index_doc.go

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@
1515
package scorch
1616

1717
import (
18-
"bytes"
1918
"reflect"
2019

2120
"github.com/RoaringBitmap/roaring/v2"
@@ -49,7 +48,7 @@ func (i *IndexSnapshotDocIDReader) Next() (index.IndexInternalID, error) {
4948
next := i.iterators[i.segmentOffset].Next()
5049
// make segment number into global number by adding offset
5150
globalOffset := i.snapshot.offsets[i.segmentOffset]
52-
return docNumberToBytes(nil, uint64(next)+globalOffset), nil
51+
return index.NewIndexInternalID(nil, uint64(next)+globalOffset), nil
5352
}
5453
return nil, nil
5554
}
@@ -63,7 +62,7 @@ func (i *IndexSnapshotDocIDReader) Advance(ID index.IndexInternalID) (index.Inde
6362
if next == nil {
6463
return nil, nil
6564
}
66-
for bytes.Compare(next, ID) < 0 {
65+
for next.Compare(ID) < 0 {
6766
next, err = i.Next()
6867
if err != nil {
6968
return nil, err

0 commit comments

Comments
 (0)