Skip to content

Commit 3c0abf2

Browse files
bbrkstorcolvingregns1
authored
[3.2.2 Backport] CBG-4495: do not put documents into rev cache for an on demand import (#7346)
* [3.2.2 Backport] CBG-4495: do not put documents into rev cache for an on demand import (#7328) * CBG-4494 do not put documents into rev cache when they are being loaded * have all on demand imports skip rev cache * skip test which requires import * remove redundant argument * Remove single duplicate test always (#7332) * CBG-4499: fix leaking goroutines (#7335) * CBG-4499: fix leaking goroutines * updates to address review --------- Co-authored-by: Tor Colvin <tor.colvin@couchbase.com> Co-authored-by: Gregory Newman-Smith <109068393+gregns1@users.noreply.github.com>
1 parent 09e9bb5 commit 3c0abf2

5 files changed

Lines changed: 137 additions & 61 deletions

File tree

db/crud.go

Lines changed: 13 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -48,12 +48,13 @@ func realDocID(docid string) string {
4848
return docid
4949
}
5050

51+
// GetDocument with raw returns the document from the bucket. This may perform an on-demand import.
5152
func (c *DatabaseCollection) GetDocument(ctx context.Context, docid string, unmarshalLevel DocumentUnmarshalLevel) (doc *Document, err error) {
5253
doc, _, err = c.GetDocumentWithRaw(ctx, docid, unmarshalLevel)
5354
return doc, err
5455
}
5556

56-
// Lowest-level method that reads a document from the bucket
57+
// GetDocumentWithRaw returns the document from the bucket. This may perform an on-demand import.
5758
func (c *DatabaseCollection) GetDocumentWithRaw(ctx context.Context, docid string, unmarshalLevel DocumentUnmarshalLevel) (doc *Document, rawBucketDoc *sgbucket.BucketDocument, err error) {
5859
key := realDocID(docid)
5960
if key == "" {
@@ -885,7 +886,8 @@ func (db *DatabaseCollectionWithUser) Put(ctx context.Context, docid string, bod
885886
}
886887

887888
allowImport := db.UseXattrs()
888-
doc, newRevID, err = db.updateAndReturnDoc(ctx, newDoc.ID, allowImport, &expiry, nil, nil, false, func(doc *Document) (resultDoc *Document, resultAttachmentData updatedAttachments, createNewRevIDSkipped bool, updatedExpiry *uint32, resultErr error) {
889+
updateRevCache := true
890+
doc, newRevID, err = db.updateAndReturnDoc(ctx, newDoc.ID, allowImport, &expiry, nil, nil, false, updateRevCache, func(doc *Document) (resultDoc *Document, resultAttachmentData updatedAttachments, createNewRevIDSkipped bool, updatedExpiry *uint32, resultErr error) {
889891
var isSgWrite bool
890892
var crc32Match bool
891893

@@ -1010,7 +1012,8 @@ func (db *DatabaseCollectionWithUser) PutExistingRevWithConflictResolution(ctx c
10101012
}
10111013

10121014
allowImport := db.UseXattrs()
1013-
doc, _, err = db.updateAndReturnDoc(ctx, newDoc.ID, allowImport, &newDoc.DocExpiry, nil, existingDoc, false, func(doc *Document) (resultDoc *Document, resultAttachmentData updatedAttachments, createNewRevIDSkipped bool, updatedExpiry *uint32, resultErr error) {
1015+
updateRevCache := true
1016+
doc, _, err = db.updateAndReturnDoc(ctx, newDoc.ID, allowImport, &newDoc.DocExpiry, nil, existingDoc, false, updateRevCache, func(doc *Document) (resultDoc *Document, resultAttachmentData updatedAttachments, createNewRevIDSkipped bool, updatedExpiry *uint32, resultErr error) {
10141017
// (Be careful: this block can be invoked multiple times if there are races!)
10151018

10161019
var isSgWrite bool
@@ -1922,7 +1925,7 @@ type updateAndReturnDocCallback func(*Document) (resultDoc *Document, resultAtta
19221925
// 2. Specify the existing document body/xattr/cas, to avoid initial retrieval of the doc in cases that the current contents are already known (e.g. import).
19231926
// On cas failure, the document will still be reloaded from the bucket as usual.
19241927
// 3. If isImport=true, document body will not be updated - only metadata xattr(s)
1925-
func (db *DatabaseCollectionWithUser) updateAndReturnDoc(ctx context.Context, docid string, allowImport bool, expiry *uint32, opts *sgbucket.MutateInOptions, existingDoc *sgbucket.BucketDocument, isImport bool, callback updateAndReturnDocCallback) (doc *Document, newRevID string, err error) {
1928+
func (db *DatabaseCollectionWithUser) updateAndReturnDoc(ctx context.Context, docid string, allowImport bool, expiry *uint32, opts *sgbucket.MutateInOptions, existingDoc *sgbucket.BucketDocument, isImport bool, updateRevCache bool, callback updateAndReturnDocCallback) (doc *Document, newRevID string, err error) {
19261929
key := realDocID(docid)
19271930
if key == "" {
19281931
return nil, "", base.HTTPErrorf(400, "Invalid doc ID")
@@ -2166,10 +2169,12 @@ func (db *DatabaseCollectionWithUser) updateAndReturnDoc(ctx context.Context, do
21662169
Deleted: doc.History[newRevID].Deleted,
21672170
}
21682171

2169-
if createNewRevIDSkipped {
2170-
db.revisionCache.Upsert(ctx, documentRevision)
2171-
} else {
2172-
db.revisionCache.Put(ctx, documentRevision)
2172+
if updateRevCache {
2173+
if createNewRevIDSkipped {
2174+
db.revisionCache.Upsert(ctx, documentRevision)
2175+
} else {
2176+
db.revisionCache.Put(ctx, documentRevision)
2177+
}
21732178
}
21742179

21752180
if db.eventMgr().HasHandlerForEvent(DocumentChange) {

db/import.go

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -146,7 +146,9 @@ func (db *DatabaseCollectionWithUser) importDoc(ctx context.Context, docid strin
146146
existingDoc.Expiry = *expiry
147147
}
148148

149-
docOut, _, err = db.updateAndReturnDoc(ctx, newDoc.ID, true, expiry, mutationOptions, existingDoc, true, func(doc *Document) (resultDocument *Document, resultAttachmentData updatedAttachments, createNewRevIDSkipped bool, updatedExpiry *uint32, resultErr error) {
149+
// do not update rev cache for on-demand imports
150+
updateRevCache := mode == ImportFromFeed
151+
docOut, _, err = db.updateAndReturnDoc(ctx, newDoc.ID, true, expiry, mutationOptions, existingDoc, true, updateRevCache, func(doc *Document) (resultDocument *Document, resultAttachmentData updatedAttachments, createNewRevIDSkipped bool, updatedExpiry *uint32, resultErr error) {
150152
// Perform cas mismatch check first, as we want to identify cas mismatch before triggering migrate handling.
151153
// If there's a cas mismatch, the doc has been updated since the version that triggered the import. Handling depends on import mode.
152154
if doc.Cas != existingDoc.Cas {

db/import_test.go

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -454,7 +454,6 @@ func TestImportWithCasFailureUpdate(t *testing.T) {
454454
assert.NoError(t, err, "Error unmarshalling body")
455455

456456
runOnce = true
457-
458457
// Trigger import
459458
_, err = collection.importDoc(ctx, testcase.docname, bodyD, nil, false, existingBucketDoc, ImportOnDemand)
460459
assert.NoError(t, err)

db/revision_cache_test.go

Lines changed: 117 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -1237,55 +1237,6 @@ func TestRevCacheOnDemand(t *testing.T) {
12371237
testCtx, testCtxCancel := context.WithCancel(base.TestCtx(t))
12381238
defer testCtxCancel()
12391239

1240-
for i := 0; i < 2; i++ {
1241-
docID := fmt.Sprintf("extraDoc%d", i)
1242-
revID, _, err := collection.Put(ctx, docID, Body{"fake": "body"})
1243-
require.NoError(t, err)
1244-
go func() {
1245-
for {
1246-
select {
1247-
case <-testCtx.Done():
1248-
return
1249-
default:
1250-
_, err = db.revisionCache.Get(ctx, docID, revID, collection.GetCollectionID(), RevCacheOmitDelta) //nolint:errcheck
1251-
}
1252-
}
1253-
}()
1254-
}
1255-
log.Printf("Updating doc to trigger on-demand import")
1256-
err = collection.dataStore.Set(docID, 0, nil, []byte(`{"ver": "2"}`))
1257-
require.NoError(t, err)
1258-
log.Printf("Calling getRev for %s, %s", docID, revID)
1259-
rev, err := collection.getRev(ctx, docID, revID, 0, nil)
1260-
require.Error(t, err)
1261-
if base.IsEnterpriseEdition() {
1262-
fmt.Println("here")
1263-
}
1264-
require.ErrorContains(t, err, "missing")
1265-
// returns empty doc rev
1266-
assert.Equal(t, "", rev.DocID)
1267-
}
1268-
1269-
func TestRevCacheOnDemandMemoryEviction(t *testing.T) {
1270-
base.SkipImportTestsIfNotEnabled(t)
1271-
1272-
dbcOptions := DatabaseContextOptions{
1273-
RevisionCacheOptions: &RevisionCacheOptions{
1274-
MaxItemCount: 20,
1275-
ShardCount: 1,
1276-
MaxBytes: 112, // equivalent to max size 2 items
1277-
},
1278-
}
1279-
db, ctx := SetupTestDBWithOptions(t, dbcOptions)
1280-
defer db.Close(ctx)
1281-
collection, ctx := GetSingleDatabaseCollectionWithUser(ctx, t, db)
1282-
docID := "doc1"
1283-
revID, _, err := collection.Put(ctx, docID, Body{"ver": "1"})
1284-
require.NoError(t, err)
1285-
1286-
testCtx, testCtxCancel := context.WithCancel(base.TestCtx(t))
1287-
defer testCtxCancel()
1288-
12891240
for i := 0; i < 2; i++ {
12901241
docID := fmt.Sprintf("extraDoc%d", i)
12911242
revID, _, err := collection.Put(ctx, docID, Body{"fake": "body"})
@@ -1310,7 +1261,6 @@ func TestRevCacheOnDemandMemoryEviction(t *testing.T) {
13101261
require.ErrorContains(t, err, "missing")
13111262
// returns empty doc rev
13121263
assert.Equal(t, "", rev.DocID)
1313-
13141264
}
13151265

13161266
func TestLoadActiveDocFromBucketRevCacheChurn(t *testing.T) {
@@ -1519,3 +1469,120 @@ func createDocAndReturnSizeAndRev(t *testing.T, ctx context.Context, docID strin
15191469

15201470
return expectedSize, rev
15211471
}
1472+
1473+
func TestRevCacheOnDemandImport(t *testing.T) {
1474+
base.SkipImportTestsIfNotEnabled(t)
1475+
1476+
dbcOptions := DatabaseContextOptions{
1477+
RevisionCacheOptions: &RevisionCacheOptions{
1478+
MaxItemCount: 2,
1479+
ShardCount: 1,
1480+
},
1481+
}
1482+
db, ctx := SetupTestDBWithOptions(t, dbcOptions)
1483+
defer db.Close(ctx)
1484+
collection, ctx := GetSingleDatabaseCollectionWithUser(ctx, t, db)
1485+
docID := "doc1"
1486+
revID, _, err := collection.Put(ctx, docID, Body{"ver": "1"})
1487+
require.NoError(t, err)
1488+
1489+
ctx, testCtxCancel := context.WithCancel(ctx)
1490+
defer testCtxCancel()
1491+
1492+
for i := 0; i < 2; i++ {
1493+
docID := fmt.Sprintf("extraDoc%d", i)
1494+
revID, _, err := collection.Put(ctx, docID, Body{"fake": "body"})
1495+
require.NoError(t, err)
1496+
go func() {
1497+
for {
1498+
select {
1499+
case <-ctx.Done():
1500+
return
1501+
default:
1502+
_, err = db.revisionCache.Get(ctx, docID, revID, collection.GetCollectionID(), RevCacheOmitDelta) //nolint:errcheck
1503+
}
1504+
}
1505+
}()
1506+
}
1507+
err = collection.dataStore.Set(docID, 0, nil, []byte(`{"ver": "2"}`))
1508+
require.NoError(t, err)
1509+
rev, err := collection.getRev(ctx, docID, revID, 0, nil)
1510+
require.Error(t, err)
1511+
require.ErrorContains(t, err, "missing")
1512+
// returns empty doc rev
1513+
assert.Equal(t, "", rev.DocID)
1514+
}
1515+
1516+
func TestRevCacheOnDemandMemoryEviction(t *testing.T) {
1517+
base.SkipImportTestsIfNotEnabled(t)
1518+
1519+
dbcOptions := DatabaseContextOptions{
1520+
RevisionCacheOptions: &RevisionCacheOptions{
1521+
MaxItemCount: 20,
1522+
ShardCount: 1,
1523+
MaxBytes: 112, // equivalent to max size 2 items
1524+
},
1525+
}
1526+
db, ctx := SetupTestDBWithOptions(t, dbcOptions)
1527+
defer db.Close(ctx)
1528+
collection, ctx := GetSingleDatabaseCollectionWithUser(ctx, t, db)
1529+
docID := "doc1"
1530+
revID, _, err := collection.Put(ctx, docID, Body{"ver": "1"})
1531+
require.NoError(t, err)
1532+
1533+
ctx, testCtxCancel := context.WithCancel(ctx)
1534+
defer testCtxCancel()
1535+
1536+
for i := 0; i < 2; i++ {
1537+
docID := fmt.Sprintf("extraDoc%d", i)
1538+
revID, _, err := collection.Put(ctx, docID, Body{"fake": "body"})
1539+
require.NoError(t, err)
1540+
go func() {
1541+
for {
1542+
select {
1543+
case <-ctx.Done():
1544+
return
1545+
default:
1546+
_, err = db.revisionCache.Get(ctx, docID, revID, collection.GetCollectionID(), RevCacheOmitDelta) //nolint:errcheck
1547+
}
1548+
}
1549+
}()
1550+
}
1551+
err = collection.dataStore.Set(docID, 0, nil, []byte(`{"ver": "2"}`))
1552+
require.NoError(t, err)
1553+
rev, err := collection.getRev(ctx, docID, revID, 0, nil)
1554+
require.Error(t, err)
1555+
require.ErrorContains(t, err, "missing")
1556+
// returns empty doc rev
1557+
assert.Equal(t, "", rev.DocID)
1558+
1559+
}
1560+
1561+
func TestRevCacheOnDemandImportNoCache(t *testing.T) {
1562+
base.SkipImportTestsIfNotEnabled(t)
1563+
1564+
db, ctx := setupTestDB(t)
1565+
defer db.Close(ctx)
1566+
collection, ctx := GetSingleDatabaseCollectionWithUser(ctx, t, db)
1567+
1568+
docID := "doc1"
1569+
revID1, _, err := collection.Put(ctx, docID, Body{"foo": "bar"})
1570+
require.NoError(t, err)
1571+
1572+
_, exists := collection.revisionCache.Peek(ctx, docID, revID1)
1573+
require.True(t, exists)
1574+
1575+
require.NoError(t, collection.dataStore.Set(docID, 0, nil, []byte(`{"foo": "baz"}`)))
1576+
1577+
doc, err := collection.GetDocument(ctx, docID, DocUnmarshalSync)
1578+
require.NoError(t, err)
1579+
require.Equal(t, Body{"foo": "baz"}, doc.Body(ctx))
1580+
1581+
// rev1 still exists in cache but not on server
1582+
_, exists = collection.revisionCache.Peek(ctx, docID, revID1)
1583+
require.True(t, exists)
1584+
1585+
// rev2 is not in cache but is on server
1586+
_, exists = collection.revisionCache.Peek(ctx, docID, doc.CurrentRev)
1587+
require.False(t, exists)
1588+
}

rest/importtest/import_test.go

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1602,14 +1602,17 @@ func TestImportRevisionCopy(t *testing.T) {
16021602
_, err := dataStore.Add(key, 0, docBody)
16031603
assert.NoError(t, err, "Unable to insert doc TestImportDelete")
16041604

1605-
// 2. Trigger import via SG retrieval
1605+
// 2. Trigger import via SG retrieval, this will not populate the rev cache.
16061606
response := rt.SendAdminRequest("GET", "/{{.keyspace}}/_raw/"+key, "")
16071607
assert.Equal(t, 200, response.Code)
16081608
var rawInsertResponse rest.RawResponse
16091609
err = base.JSONUnmarshal(response.Body.Bytes(), &rawInsertResponse)
16101610
assert.NoError(t, err, "Unable to unmarshal raw response")
16111611
rev1id := rawInsertResponse.Sync.Rev
16121612

1613+
// Populate rev cache by getting the doc again
1614+
rt.GetDoc(key)
1615+
16131616
// 3. Update via SDK
16141617
updatedBody := make(map[string]interface{})
16151618
updatedBody["test"] = "TestImportRevisionCopyModified"

0 commit comments

Comments
 (0)