⚡ Improve attribute view search cache coordination #18474

Author: 88250

kernel/av/av.go

@@ -35,6 +35,7 @@ import (
 	"github.com/siyuan-note/logging"
 	"github.com/siyuan-note/siyuan/kernel/cache"
 	"github.com/siyuan-note/siyuan/kernel/util"
+	"golang.org/x/sync/singleflight"
 )
 
 // AttributeView 描述了属性视图的结构。
@@ -592,48 +593,70 @@ func parseAttributeViewSearchInfo(data []byte) (ret *AttributeViewSearchInfo, er
 	return
 }
 
+var attributeViewSearchInfoFlight singleflight.Group
+
 func GetAttributeViewSearchInfoInBox(avID, boxID string) (ret *AttributeViewSearchInfo, err error) {
 	avJSONPath, avBoxID := FindAttributeViewPathInBox(avID, boxID)
 	if avJSONPath == "" {
 		return
 	}
+	if cached, ok := cache.GetAVSearchDataInBox[*AttributeViewSearchInfo](avID, avBoxID); ok {
+		return cached, nil
+	}
+
+	value, err, _ := attributeViewSearchInfoFlight.Do(avBoxID+"\x00"+avID, func() (any, error) {
+		return loadAttributeViewSearchInfoInBox(avID, avJSONPath, avBoxID)
+	})
+	if value != nil {
+		ret = value.(*AttributeViewSearchInfo)
+	}
+	return
+}
+
+func loadAttributeViewSearchInfoInBox(avID, avJSONPath, avBoxID string) (ret *AttributeViewSearchInfo, err error) {
 	release, lockErr := holdAVBoxReadLock(avBoxID)
 	if lockErr != nil {
 		return nil, lockErr
 	}
 	defer release()
 
-	if cached, ok := cache.GetAVSearchDataInBox[*AttributeViewSearchInfo](avID, avBoxID); ok {
-		return cached, nil
-	}
-
-	var data []byte
-	var dataVersion uint64
-	if cached, version, ok := cache.GetAVDataWithVersionInBox(avID, avBoxID); ok {
-		data = cached
-		dataVersion = version
-	} else {
-		dataVersion = cache.EnsureAVDataVersionInBox(avID, avBoxID)
-		if data, err = filelock.ReadFile(avJSONPath); err != nil {
-			logging.LogErrorf("read attribute view [%s] failed: %s", avJSONPath, err)
-			return
+	for i := 0; i < 3; i++ {
+		if cached, ok := cache.GetAVSearchDataInBox[*AttributeViewSearchInfo](avID, avBoxID); ok {
+			return cached, nil
 		}
-		if avBoxID != "" {
-			if data, err = decryptAVDataLocked(avBoxID, avID, data); err != nil {
-				logging.LogErrorf("decrypt attribute view [%s] failed: %s", avJSONPath, err)
+
+		var data []byte
+		var dataVersion uint64
+		if cached, version, ok := cache.GetAVDataWithVersionInBox(avID, avBoxID); ok {
+			data = cached
+			dataVersion = version
+		} else {
+			dataVersion = cache.EnsureAVDataVersionInBox(avID, avBoxID)
+			if data, err = filelock.ReadFile(avJSONPath); err != nil {
+				logging.LogErrorf("read attribute view [%s] failed: %s", avJSONPath, err)
 				return
 			}
-		} else if util.IsCiphertext(data) {
-			return
+			if avBoxID != "" {
+				if data, err = decryptAVDataLocked(avBoxID, avID, data); err != nil {
+					logging.LogErrorf("decrypt attribute view [%s] failed: %s", avJSONPath, err)
+					return
+				}
+			} else if util.IsCiphertext(data) {
+				return nil, nil
+			}
 		}
-	}
 
-	if ret, err = parseAttributeViewSearchInfo(data); err != nil {
-		logging.LogErrorf("unmarshal attribute view search info [%s] failed: %s", avID, err)
-		return nil, err
+		if ret, err = parseAttributeViewSearchInfo(data); err != nil {
+			logging.LogErrorf("unmarshal attribute view search info [%s] failed: %s", avID, err)
+			return nil, err
+		}
+		if cache.SetAVSearchDataInBox(avID, avBoxID, dataVersion, ret) {
+			return ret, nil
+		}
 	}
-	cache.SetAVSearchDataInBox(avID, avBoxID, dataVersion, ret)
-	return
+	err = fmt.Errorf("attribute view [%s] changed while loading search info", avID)
+	logging.LogWarnf("%s", err)
+	return nil, err
 }
 
 func GetAttributeViewContent(avID string) (content string) {

kernel/cache/av.go

@@ -18,6 +18,7 @@ package cache
 
 import (
 	"sync"
+	"sync/atomic"
 
 	"github.com/dgraph-io/ristretto"
 )
@@ -45,6 +46,7 @@ var avDataVersion uint64
 var avDataVersions = map[string]uint64{}
 var avSearchDataCache = map[string]*avSearchDataEntry{}
 var avSearchDataCacheLock sync.RWMutex
+var avCacheGeneration atomic.Uint64
 
 func avCacheKey(avID, boxID string) string {
 	return boxID + "\x00" + avID
@@ -143,6 +145,7 @@ func RemoveAVData(avID string) {
 }
 
 func ClearAVCache() {
+	avCacheGeneration.Add(1)
 	avCacheKeysLock.Lock()
 	avCacheKeys = map[string]map[string]struct{}{}
 	avCacheKeysLock.Unlock()
@@ -154,6 +157,10 @@ func ClearAVCache() {
 	avSearchDataCacheLock.Unlock()
 }
 
+func GetAVCacheGeneration() uint64 {
+	return avCacheGeneration.Load()
+}
+
 func GetAVSearchDataInBox[T any](avID, boxID string) (ret T, ok bool) {
 	avSearchDataCacheLock.RLock()
 	key := avCacheKey(avID, boxID)

kernel/cache/av_test.go

@@ -73,3 +73,11 @@ func TestAVSearchDataWithoutRawData(t *testing.T) {
 		t.Fatal("setting raw data should invalidate standalone search data")
 	}
 }
+
+func TestAVCacheGeneration(t *testing.T) {
+	generation := GetAVCacheGeneration()
+	ClearAVCache()
+	if current := GetAVCacheGeneration(); current != generation+1 {
+		t.Fatalf("unexpected AV cache generation: %d", current)
+	}
+}

kernel/model/attribute_view.go

@@ -18,6 +18,7 @@ package model
 
 import (
 	"bytes"
+	"context"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -2743,27 +2744,108 @@ type SearchAttributeViewOptions struct {
 	IncludeViewMatches bool
 }
 
-var attributeViewSearchCacheWarmups sync.Map
+type attributeViewSearchCacheWarmup struct {
+	signature uint64
+	cancel    context.CancelFunc
+	running   bool
+}
+
+var attributeViewSearchCacheWarmups = struct {
+	sync.Mutex
+	states map[string]*attributeViewSearchCacheWarmup
+}{states: map[string]*attributeViewSearchCacheWarmup{}}
+
+var attributeViewSearchCacheWarmupDelay = 500 * time.Millisecond
+var loadAttributeViewSearchInfo = av.GetAttributeViewSearchInfoInBox
 
-func warmAttributeViewSearchCache(boxID string, avIDs []string) {
+const attributeViewSearchCacheWarmupHashOffset = uint64(1469598103934665603)
+const attributeViewSearchCacheWarmupHashPrime = uint64(1099511628211)
+
+func warmAttributeViewSearchCache(boxID string, avIDs []string, signature uint64) {
 	if len(avIDs) == 0 {
 		return
 	}
-	if _, loaded := attributeViewSearchCacheWarmups.LoadOrStore(boxID, struct{}{}); loaded {
+
+	attributeViewSearchCacheWarmups.Lock()
+	if state := attributeViewSearchCacheWarmups.states[boxID]; state != nil && state.signature == signature {
+		attributeViewSearchCacheWarmups.Unlock()
 		return
 	}
+	if state := attributeViewSearchCacheWarmups.states[boxID]; state != nil && state.cancel != nil {
+		state.cancel()
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+	state := &attributeViewSearchCacheWarmup{signature: signature, cancel: cancel, running: true}
+	attributeViewSearchCacheWarmups.states[boxID] = state
+	attributeViewSearchCacheWarmups.Unlock()
+
 	ids := slices.Clone(avIDs)
+	delay := attributeViewSearchCacheWarmupDelay
+	loader := loadAttributeViewSearchInfo
 	go func() {
-		defer attributeViewSearchCacheWarmups.Delete(boxID)
+		completed := false
+		defer func() {
+			attributeViewSearchCacheWarmups.Lock()
+			defer attributeViewSearchCacheWarmups.Unlock()
+			if attributeViewSearchCacheWarmups.states[boxID] != state {
+				return
+			}
+			if completed {
+				state.cancel = nil
+				state.running = false
+			} else {
+				delete(attributeViewSearchCacheWarmups.states, boxID)
+			}
+		}()
+
+		timer := time.NewTimer(delay)
+		defer timer.Stop()
+		select {
+		case <-ctx.Done():
+			return
+		case <-timer.C:
+		}
+
 		for _, avID := range ids {
+			select {
+			case <-ctx.Done():
+				return
+			default:
+			}
 			if isSyncingStorages() {
 				return
 			}
-			_, _ = av.GetAttributeViewSearchInfoInBox(avID, boxID)
+			_, _ = loader(avID, boxID)
 		}
+		completed = true
 	}()
 }
 
+func stopAttributeViewSearchCacheWarmup(boxID string) {
+	attributeViewSearchCacheWarmups.Lock()
+	defer attributeViewSearchCacheWarmups.Unlock()
+	state := attributeViewSearchCacheWarmups.states[boxID]
+	if state == nil || !state.running {
+		return
+	}
+	state.cancel()
+	delete(attributeViewSearchCacheWarmups.states, boxID)
+}
+
+func updateAttributeViewSearchCacheWarmupSignature(signature uint64, avID string, info os.FileInfo) uint64 {
+	for i := 0; i < len(avID); i++ {
+		signature ^= uint64(avID[i])
+		signature *= attributeViewSearchCacheWarmupHashPrime
+	}
+	if info != nil {
+		signature ^= uint64(info.ModTime().UnixNano())
+		signature *= attributeViewSearchCacheWarmupHashPrime
+		signature ^= uint64(info.Size())
+		signature *= attributeViewSearchCacheWarmupHashPrime
+	}
+	return signature
+}
+
 func matchAttributeViewSearchName(name string, keywords []string) (score float64, hit bool) {
 	if name == "" || len(keywords) == 0 {
 		return
@@ -2781,6 +2863,23 @@ func matchAttributeViewSearchName(name string, keywords []string) (score float64
 	return
 }
 
+func sortAndLimitAttributeViewSearchResults(results []*AvSearchTempResult, keyword string) []*AvSearchTempResult {
+	if keyword == "" {
+		sort.Slice(results, func(i, j int) bool { return results[i].AvUpdated > results[j].AvUpdated })
+	} else {
+		sort.SliceStable(results, func(i, j int) bool {
+			if results[i].Score == results[j].Score {
+				return results[i].AvUpdated > results[j].AvUpdated
+			}
+			return results[i].Score > results[j].Score
+		})
+	}
+	if 12 < len(results) {
+		return results[:12]
+	}
+	return results
+}
+
 func SearchAttributeView(keyword string, excludeAvIDs []string, currentAvID, currentBlockID string) []*AvSearchResult {
 	return SearchAttributeViewWithOptions(SearchAttributeViewOptions{
 		Keyword:        keyword,
@@ -2823,6 +2922,9 @@ func SearchAttributeViewWithOptions(options SearchAttributeViewOptions) (ret []*
 
 	var avSearchTmpResults []*AvSearchTempResult
 	var warmupAvIDs []string
+	warmupSignature := attributeViewSearchCacheWarmupHashOffset
+	warmupSignature ^= cache.GetAVCacheGeneration()
+	warmupSignature *= attributeViewSearchCacheWarmupHashPrime
 	boxID := ""
 	if options.CurrentBlockID != "" {
 		if bt := treenode.GetBlockTree(options.CurrentBlockID); nil != bt && IsEncryptedBox(bt.BoxID) {
@@ -2831,6 +2933,9 @@ func SearchAttributeViewWithOptions(options SearchAttributeViewOptions) (ret []*
 	} else if options.CurrentAvID != "" {
 		_, boxID = av.FindAttributeViewPath(options.CurrentAvID)
 	}
+	if keyword != "" {
+		stopAttributeViewSearchCacheWarmup(boxID)
+	}
 	avDir := filepath.Join(util.DataDir, "storage", "av")
 	if boxID != "" {
 		avDir = filepath.Join(util.DataDir, boxID, "storage", "av")
@@ -2866,16 +2971,17 @@ func SearchAttributeViewWithOptions(options SearchAttributeViewOptions) (ret []*
 		if nil == avBlockRels[id] {
 			continue
 		}
+		readFileInfoStart := time.Now()
+		info, _ := entry.Info()
+		readFileInfoElapsed += time.Since(readFileInfoStart)
 		warmupAvIDs = append(warmupAvIDs, id)
+		warmupSignature = updateAttributeViewSearchCacheWarmupSignature(warmupSignature, id, info)
 
 		if gulu.Str.Contains(id, options.ExcludeAvIDs) {
 			continue
 		}
 		eligibleFileCount++
 
-		readFileInfoStart := time.Now()
-		info, _ := entry.Info()
-		readFileInfoElapsed += time.Since(readFileInfoStart)
 		if info != nil {
 			eligibleFileSize += info.Size()
 		}
@@ -2928,20 +3034,8 @@ func SearchAttributeViewWithOptions(options SearchAttributeViewOptions) (ret []*
 	scanElapsed = time.Since(scanStart)
 
 	sortStart := time.Now()
-	if "" == keyword {
-		sort.Slice(avSearchTmpResults, func(i, j int) bool { return avSearchTmpResults[i].AvUpdated > avSearchTmpResults[j].AvUpdated })
-	} else {
-		sort.SliceStable(avSearchTmpResults, func(i, j int) bool {
-			if avSearchTmpResults[i].Score == avSearchTmpResults[j].Score {
-				return avSearchTmpResults[i].AvUpdated > avSearchTmpResults[j].AvUpdated
-			}
-			return avSearchTmpResults[i].Score > avSearchTmpResults[j].Score
-		})
-	}
 	matchedCount = len(avSearchTmpResults)
-	if 12 <= len(avSearchTmpResults) {
-		avSearchTmpResults = avSearchTmpResults[:12]
-	}
+	avSearchTmpResults = sortAndLimitAttributeViewSearchResults(avSearchTmpResults, keyword)
 	sortElapsed = time.Since(sortStart)
 
 	resolveStart := time.Now()
@@ -3022,7 +3116,7 @@ func SearchAttributeViewWithOptions(options SearchAttributeViewOptions) (ret []*
 	}
 	resolveElapsed = time.Since(resolveStart)
 	if keyword == "" {
-		warmAttributeViewSearchCache(boxID, warmupAvIDs)
+		warmAttributeViewSearchCache(boxID, warmupAvIDs, warmupSignature)
 	}
 	return
 }