Add recency bias to semantic search (timestamps in prompts + time-decay reranking)

api/api_search.go

@@ -137,6 +137,7 @@ type RawSearchResult struct {
 	Username    string  `json:"username"`
 	Content     string  `json:"content"`
 	Score       float32 `json:"score"`
+	CreateAt    int64   `json:"create_at"` // Post creation timestamp (Unix millis)
 }
 
 // RawSearchResponse represents the response body for the raw semantic search endpoint
@@ -220,6 +221,7 @@ func (a *API) handleRawSearch(c *gin.Context) {
 			Username:    r.Username,
 			Content:     r.Content,
 			Score:       r.Score,
+			CreateAt:    r.CreateAt,
 		})
 	}
 

embeddings/composite.go

@@ -6,6 +6,7 @@ package embeddings
 import (
 	"context"
 	"fmt"
+	"time"
 
 	"github.com/mattermost/mattermost-plugin-agents/v2/chunking"
 )
@@ -15,14 +16,16 @@ type CompositeSearch struct {
 	store    VectorStore
 	provider EmbeddingProvider
 	options  chunking.Options
+	recency  RecencyBiasSettings
 }
 
 // NewCompositeSearch creates a new CompositeSearch with required chunking options
-func NewCompositeSearch(store VectorStore, provider EmbeddingProvider, options chunking.Options) *CompositeSearch {
+func NewCompositeSearch(store VectorStore, provider EmbeddingProvider, options chunking.Options, recency RecencyBiasSettings) *CompositeSearch {
 	return &CompositeSearch{
 		store:    store,
 		provider: provider,
 		options:  options,
+		recency:  recency,
 	}
 }
 
@@ -77,12 +80,41 @@ func (c *CompositeSearch) Search(ctx context.Context, query string, opts SearchO
 		return nil, err
 	}
 
-	// Search for matching chunks
-	results, err := c.store.Search(ctx, embedding, opts)
+	if !c.recency.Enabled {
+		return c.store.Search(ctx, embedding, opts)
+	}
+
+	// Recency bias: over-fetch candidates by pure similarity (keeps the ANN
+	// index usable), rerank in memory with the time decay, then apply the
+	// caller's offset/limit window to the reranked order.
+	fetchOpts := opts
+	fetchOpts.Offset = 0
+	fetchOpts.Limit = recencyFetchLimit(opts.Limit, opts.Offset)
+
+	// The store caps a single search at MaxSearchResults rows, so a window
+	// that extends past the cap cannot be served from the reranked candidate
+	// pool. Fall back to store-side pagination in raw similarity order rather
+	// than silently truncating deep pages.
+	if fetchOpts.Limit > MaxSearchResults || (fetchOpts.Limit == 0 && opts.Offset > 0) {
+		return c.store.Search(ctx, embedding, opts)
+	}
+
+	results, err := c.store.Search(ctx, embedding, fetchOpts)
 	if err != nil {
 		return nil, err
 	}
 
+	rerankByRecency(results, time.Now().UnixMilli(), c.recency)
+
+	if opts.Offset > 0 {
+		if opts.Offset >= len(results) {
+			return nil, nil
+		}
+		results = results[opts.Offset:]
+	}
+	if opts.Limit > 0 && len(results) > opts.Limit {
+		results = results[:opts.Limit]
+	}
 	return results, nil
 }
 

embeddings/composite_test.go

@@ -7,6 +7,7 @@ import (
 	"context"
 	"errors"
 	"testing"
+	"time"
 
 	"github.com/mattermost/mattermost-plugin-agents/v2/chunking"
 	"github.com/stretchr/testify/assert"
@@ -277,7 +278,7 @@ func TestCompositeSearch_Store(t *testing.T) {
 			store := &stubVectorStore{storeFunc: tt.storeFunc}
 			provider := &stubEmbeddingProvider{batchCreateEmbeddingsFunc: tt.batchEmbedFunc}
 
-			cs := NewCompositeSearch(store, provider, tt.options)
+			cs := NewCompositeSearch(store, provider, tt.options, RecencyBiasSettings{})
 
 			err := cs.Store(context.Background(), tt.docs)
 
@@ -420,7 +421,7 @@ func TestCompositeSearch_Search(t *testing.T) {
 			store := &stubVectorStore{searchFunc: tt.searchFunc}
 			provider := &stubEmbeddingProvider{createEmbeddingFunc: tt.createEmbedFunc}
 
-			cs := NewCompositeSearch(store, provider, defaultOptions)
+			cs := NewCompositeSearch(store, provider, defaultOptions, RecencyBiasSettings{})
 
 			results, err := cs.Search(context.Background(), tt.query, tt.searchOpts)
 
@@ -441,6 +442,156 @@ func TestCompositeSearch_Search(t *testing.T) {
 	}
 }
 
+func TestCompositeSearch_RecencyBias(t *testing.T) {
+	defaultOptions := chunking.Options{
+		ChunkSize:        1000,
+		ChunkOverlap:     200,
+		ChunkingStrategy: "sentences",
+	}
+	enabled := RecencyBiasSettings{Enabled: true, HalfLifeDays: 7, Floor: 0.7}
+
+	now := time.Now().UnixMilli()
+	createAtDaysAgo := func(days int64) int64 { return now - days*millisPerDay }
+
+	// Raw similarity order: old-strong, fresh-mid, fresh-weak.
+	// Adjusted (half-life 7d, floor 0.7): old-strong 0.80*~0.70=~0.56,
+	// fresh-mid ~0.78, fresh-weak ~0.60 -> fresh-mid, fresh-weak, old-strong.
+	defaultStoreResults := func() []SearchResult {
+		return []SearchResult{
+			{Document: PostDocument{PostID: "old-strong", CreateAt: createAtDaysAgo(60)}, Score: 0.80},
+			{Document: PostDocument{PostID: "fresh-mid", CreateAt: createAtDaysAgo(0)}, Score: 0.78},
+			{Document: PostDocument{PostID: "fresh-weak", CreateAt: createAtDaysAgo(0)}, Score: 0.60},
+		}
+	}
+
+	tests := []struct {
+		name                string
+		recency             RecencyBiasSettings
+		searchOpts          SearchOptions
+		storeResults        func() []SearchResult
+		expectedFetchLimit  int
+		expectedFetchOffset int
+		expectedOrder       []string
+	}{
+		{
+			name:                "disabled passes options through and preserves store order",
+			recency:             RecencyBiasSettings{},
+			searchOpts:          SearchOptions{Limit: 5, Offset: 2},
+			storeResults:        defaultStoreResults,
+			expectedFetchLimit:  5,
+			expectedFetchOffset: 2,
+			expectedOrder:       []string{"old-strong", "fresh-mid", "fresh-weak"},
+		},
+		{
+			name:                "over-fetches, reranks by recency, truncates to limit",
+			recency:             enabled,
+			searchOpts:          SearchOptions{Limit: 2},
+			storeResults:        defaultStoreResults,
+			expectedFetchLimit:  recencyMinCandidates,
+			expectedFetchOffset: 0,
+			expectedOrder:       []string{"fresh-mid", "fresh-weak"},
+		},
+		{
+			name:       "floor keeps old canonical answer above weak fresh result",
+			recency:    enabled,
+			searchOpts: SearchOptions{Limit: 2},
+			storeResults: func() []SearchResult {
+				return []SearchResult{
+					{Document: PostDocument{PostID: "old-canonical", CreateAt: createAtDaysAgo(365)}, Score: 0.90},
+					{Document: PostDocument{PostID: "weak-fresh", CreateAt: createAtDaysAgo(0)}, Score: 0.50},
+				}
+			},
+			expectedFetchLimit:  recencyMinCandidates,
+			expectedFetchOffset: 0,
+			expectedOrder:       []string{"old-canonical", "weak-fresh"},
+		},
+		{
+			name:                "offset applies to the reranked order",
+			recency:             enabled,
+			searchOpts:          SearchOptions{Limit: 1, Offset: 1},
+			storeResults:        defaultStoreResults,
+			expectedFetchLimit:  recencyMinCandidates + 1,
+			expectedFetchOffset: 0,
+			expectedOrder:       []string{"fresh-weak"},
+		},
+		{
+			name:                "offset beyond candidates returns no results",
+			recency:             enabled,
+			searchOpts:          SearchOptions{Limit: 5, Offset: 10},
+			storeResults:        defaultStoreResults,
+			expectedFetchLimit:  recencyMinCandidates + 10,
+			expectedFetchOffset: 0,
+			expectedOrder:       []string{},
+		},
+		{
+			// The window extends past the store's hard cap, so reranking
+			// cannot serve it; pagination must fall through to the store
+			// (raw similarity order) instead of returning a truncated page.
+			name:                "window beyond store cap falls back to store pagination",
+			recency:             enabled,
+			searchOpts:          SearchOptions{Limit: 5, Offset: MaxSearchResults},
+			storeResults:        defaultStoreResults,
+			expectedFetchLimit:  5,
+			expectedFetchOffset: MaxSearchResults,
+			expectedOrder:       []string{"old-strong", "fresh-mid", "fresh-weak"},
+		},
+		{
+			name:                "unbounded limit with offset falls back to store pagination",
+			recency:             enabled,
+			searchOpts:          SearchOptions{Limit: 0, Offset: 5},
+			storeResults:        defaultStoreResults,
+			expectedFetchLimit:  0,
+			expectedFetchOffset: 5,
+			expectedOrder:       []string{"old-strong", "fresh-mid", "fresh-weak"},
+		},
+		{
+			name:                "unbounded limit without offset reranks everything",
+			recency:             enabled,
+			searchOpts:          SearchOptions{Limit: 0},
+			storeResults:        defaultStoreResults,
+			expectedFetchLimit:  0,
+			expectedFetchOffset: 0,
+			expectedOrder:       []string{"fresh-mid", "fresh-weak", "old-strong"},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			rawScores := make(map[string]float32)
+			for _, r := range tt.storeResults() {
+				rawScores[r.Document.PostID] = r.Score
+			}
+
+			store := &stubVectorStore{
+				searchFunc: func(ctx context.Context, embedding []float32, opts SearchOptions) ([]SearchResult, error) {
+					return tt.storeResults(), nil
+				},
+			}
+			provider := &stubEmbeddingProvider{}
+			cs := NewCompositeSearch(store, provider, defaultOptions, tt.recency)
+
+			results, err := cs.Search(context.Background(), "query", tt.searchOpts)
+			require.NoError(t, err)
+
+			require.Len(t, store.searchCalls, 1)
+			assert.Equal(t, tt.expectedFetchLimit, store.searchCalls[0].opts.Limit, "store fetch limit")
+			assert.Equal(t, tt.expectedFetchOffset, store.searchCalls[0].opts.Offset, "store fetch offset")
+
+			order := make([]string, 0, len(results))
+			for _, r := range results {
+				order = append(order, r.Document.PostID)
+			}
+			assert.Equal(t, tt.expectedOrder, order)
+
+			// The returned Score stays the raw similarity; recency only reorders.
+			for _, r := range results {
+				assert.Equal(t, rawScores[r.Document.PostID], r.Score,
+					"raw score of %s must be preserved", r.Document.PostID)
+			}
+		})
+	}
+}
+
 func TestCompositeSearch_Delete(t *testing.T) {
 	defaultOptions := chunking.Options{
 		ChunkSize:        1000,
@@ -509,7 +660,7 @@ func TestCompositeSearch_Delete(t *testing.T) {
 			store := &stubVectorStore{deleteFunc: tt.deleteFunc}
 			provider := &stubEmbeddingProvider{}
 
-			cs := NewCompositeSearch(store, provider, defaultOptions)
+			cs := NewCompositeSearch(store, provider, defaultOptions, RecencyBiasSettings{})
 
 			err := cs.Delete(context.Background(), tt.postIDs)
 
@@ -575,7 +726,7 @@ func TestCompositeSearch_Clear(t *testing.T) {
 			store := &stubVectorStore{clearFunc: tt.clearFunc}
 			provider := &stubEmbeddingProvider{}
 
-			cs := NewCompositeSearch(store, provider, defaultOptions)
+			cs := NewCompositeSearch(store, provider, defaultOptions, RecencyBiasSettings{})
 
 			err := cs.Clear(context.Background())
 
@@ -616,7 +767,7 @@ func TestCompositeSearch_ContextCancellation(t *testing.T) {
 			},
 		}
 
-		cs := NewCompositeSearch(store, provider, defaultOptions)
+		cs := NewCompositeSearch(store, provider, defaultOptions, RecencyBiasSettings{})
 
 		ctx, cancel := context.WithCancel(context.Background())
 		cancel() // Cancel immediately
@@ -641,7 +792,7 @@ func TestCompositeSearch_ContextCancellation(t *testing.T) {
 			},
 		}
 
-		cs := NewCompositeSearch(store, provider, defaultOptions)
+		cs := NewCompositeSearch(store, provider, defaultOptions, RecencyBiasSettings{})
 
 		ctx, cancel := context.WithCancel(context.Background())
 		cancel() // Cancel immediately

embeddings/embeddings.go

@@ -47,6 +47,10 @@ type SearchResult struct {
 	Score    float32
 }
 
+// MaxSearchResults is the hard cap on rows a vector store returns for a
+// single search; stores clamp unset (<=0) or larger limits to this value.
+const MaxSearchResults = 1000
+
 // SearchOptions contains parameters for search operations
 type SearchOptions struct {
 	Limit         int
@@ -159,6 +163,9 @@ type EmbeddingSearchConfig struct {
 	ReindexWorkers       int              `json:"reindexWorkers,omitempty"`
 	ReindexBatchSize     int              `json:"reindexBatchSize,omitempty"`
 	ReindexIndexStrategy string           `json:"reindexIndexStrategy,omitempty"`
+	RecencyBiasEnabled   bool             `json:"recencyBiasEnabled,omitempty"`
+	RecencyHalfLifeDays  float64          `json:"recencyHalfLifeDays,omitempty"`
+	RecencyFloor         float64          `json:"recencyFloor,omitempty"`
 }
 
 // GetReindexWorkers returns the configured reindex worker count, clamped to
@@ -179,6 +186,25 @@ func (c *EmbeddingSearchConfig) GetReindexBatchSize() int {
 	return min(c.ReindexBatchSize, MaxReindexBatchSize)
 }
 
+// GetRecencyBiasSettings resolves the recency bias config: unset (<=0)
+// half-life and floor fall back to defaults; the floor is clamped to [0, 1].
+// These are query-time-only settings; changing them never requires a reindex.
+func (c *EmbeddingSearchConfig) GetRecencyBiasSettings() RecencyBiasSettings {
+	settings := RecencyBiasSettings{
+		Enabled:      c.RecencyBiasEnabled,
+		HalfLifeDays: c.RecencyHalfLifeDays,
+		Floor:        c.RecencyFloor,
+	}
+	if settings.HalfLifeDays <= 0 {
+		settings.HalfLifeDays = DefaultRecencyHalfLifeDays
+	}
+	if settings.Floor <= 0 {
+		settings.Floor = DefaultRecencyFloor
+	}
+	settings.Floor = min(settings.Floor, 1)
+	return settings
+}
+
 // EffectiveReindexIndexStrategy: defer if set, otherwise maintain.
 func (c *EmbeddingSearchConfig) EffectiveReindexIndexStrategy() string {
 	switch c.ReindexIndexStrategy {