Skip to content

Commit 755cd00

Browse files
keegancsmithkeegancsmith
authored
fix/search: distinguish cancellation-attributable file skips (#1121)
FilesSkipped currently combines match-limit truncation with context cancellation, preventing consumers from interpreting incomplete-search statistics safely. Preserve that compatibility total while exposing an additive cancellation subset through API transports and observability.
1 parent 36cb53c commit 755cd00

9 files changed

Lines changed: 254 additions & 198 deletions

File tree

‎api.go‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -395,10 +395,15 @@ type Stats struct {
395395
// Files for which we loaded file content to verify substring matches
396396
FilesLoaded int
397397

398-
// Candidate files whose contents weren't examined because we
399-
// gathered enough matches.
398+
// Candidate files whose contents weren't examined because a search limit
399+
// was reached or the search context was canceled.
400400
FilesSkipped int
401401

402+
// FilesSkippedDueToCancellation is the portion of FilesSkipped caused by a
403+
// context cancellation observed while searching a shard. This includes
404+
// cancellations caused by deadlines and internal search limits.
405+
FilesSkippedDueToCancellation int
406+
402407
// Shards that we scanned to find matches.
403408
ShardsScanned int
404409

@@ -437,7 +442,7 @@ type Stats struct {
437442
}
438443

439444
func (s *Stats) sizeBytes() (sz uint64) {
440-
sz = 16 * 8 // This assumes we are running on a 64-bit architecture
445+
sz = 17 * 8 // This assumes we are running on a 64-bit architecture
441446
sz += 1 // FlushReason
442447

443448
return
@@ -451,6 +456,7 @@ func (s *Stats) Add(o Stats) {
451456
s.FilesConsidered += o.FilesConsidered
452457
s.FilesLoaded += o.FilesLoaded
453458
s.FilesSkipped += o.FilesSkipped
459+
s.FilesSkippedDueToCancellation += o.FilesSkippedDueToCancellation
454460
s.MatchCount += o.MatchCount
455461
s.NgramMatches += o.NgramMatches
456462
s.NgramLookups += o.NgramLookups
@@ -483,6 +489,7 @@ func (s *Stats) Zero() bool {
483489
s.FilesConsidered > 0 ||
484490
s.FilesLoaded > 0 ||
485491
s.FilesSkipped > 0 ||
492+
s.FilesSkippedDueToCancellation > 0 ||
486493
s.MatchCount > 0 ||
487494
s.NgramMatches > 0 ||
488495
s.NgramLookups > 0 ||

‎api_proto.go‎

Lines changed: 42 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -286,51 +286,53 @@ func (fr FlushReason) Generate(rand *rand.Rand, size int) reflect.Value {
286286

287287
func StatsFromProto(p *webserverv1.Stats) Stats {
288288
return Stats{
289-
ContentBytesLoaded: p.GetContentBytesLoaded(),
290-
IndexBytesLoaded: p.GetIndexBytesLoaded(),
291-
Crashes: int(p.GetCrashes()),
292-
Duration: p.GetDuration().AsDuration(),
293-
FileCount: int(p.GetFileCount()),
294-
ShardFilesConsidered: int(p.GetShardFilesConsidered()),
295-
FilesConsidered: int(p.GetFilesConsidered()),
296-
FilesLoaded: int(p.GetFilesLoaded()),
297-
FilesSkipped: int(p.GetFilesSkipped()),
298-
ShardsScanned: int(p.GetShardsScanned()),
299-
ShardsSkipped: int(p.GetShardsSkipped()),
300-
ShardsSkippedFilter: int(p.GetShardsSkippedFilter()),
301-
MatchCount: int(p.GetMatchCount()),
302-
NgramMatches: int(p.GetNgramMatches()),
303-
NgramLookups: int(p.GetNgramLookups()),
304-
Wait: p.GetWait().AsDuration(),
305-
MatchTreeConstruction: p.GetMatchTreeConstruction().AsDuration(),
306-
MatchTreeSearch: p.GetMatchTreeSearch().AsDuration(),
307-
RegexpsConsidered: int(p.GetRegexpsConsidered()),
308-
FlushReason: FlushReasonFromProto(p.GetFlushReason()),
289+
ContentBytesLoaded: p.GetContentBytesLoaded(),
290+
IndexBytesLoaded: p.GetIndexBytesLoaded(),
291+
Crashes: int(p.GetCrashes()),
292+
Duration: p.GetDuration().AsDuration(),
293+
FileCount: int(p.GetFileCount()),
294+
ShardFilesConsidered: int(p.GetShardFilesConsidered()),
295+
FilesConsidered: int(p.GetFilesConsidered()),
296+
FilesLoaded: int(p.GetFilesLoaded()),
297+
FilesSkipped: int(p.GetFilesSkipped()),
298+
FilesSkippedDueToCancellation: int(p.GetFilesSkippedDueToCancellation()),
299+
ShardsScanned: int(p.GetShardsScanned()),
300+
ShardsSkipped: int(p.GetShardsSkipped()),
301+
ShardsSkippedFilter: int(p.GetShardsSkippedFilter()),
302+
MatchCount: int(p.GetMatchCount()),
303+
NgramMatches: int(p.GetNgramMatches()),
304+
NgramLookups: int(p.GetNgramLookups()),
305+
Wait: p.GetWait().AsDuration(),
306+
MatchTreeConstruction: p.GetMatchTreeConstruction().AsDuration(),
307+
MatchTreeSearch: p.GetMatchTreeSearch().AsDuration(),
308+
RegexpsConsidered: int(p.GetRegexpsConsidered()),
309+
FlushReason: FlushReasonFromProto(p.GetFlushReason()),
309310
}
310311
}
311312

312313
func (s *Stats) ToProto() *webserverv1.Stats {
313314
return &webserverv1.Stats{
314-
ContentBytesLoaded: s.ContentBytesLoaded,
315-
IndexBytesLoaded: s.IndexBytesLoaded,
316-
Crashes: int64(s.Crashes),
317-
Duration: durationpb.New(s.Duration),
318-
FileCount: int64(s.FileCount),
319-
ShardFilesConsidered: int64(s.ShardFilesConsidered),
320-
FilesConsidered: int64(s.FilesConsidered),
321-
FilesLoaded: int64(s.FilesLoaded),
322-
FilesSkipped: int64(s.FilesSkipped),
323-
ShardsScanned: int64(s.ShardsScanned),
324-
ShardsSkipped: int64(s.ShardsSkipped),
325-
ShardsSkippedFilter: int64(s.ShardsSkippedFilter),
326-
MatchCount: int64(s.MatchCount),
327-
NgramMatches: int64(s.NgramMatches),
328-
NgramLookups: int64(s.NgramLookups),
329-
Wait: durationpb.New(s.Wait),
330-
MatchTreeConstruction: durationpb.New(s.MatchTreeConstruction),
331-
MatchTreeSearch: durationpb.New(s.MatchTreeSearch),
332-
RegexpsConsidered: int64(s.RegexpsConsidered),
333-
FlushReason: s.FlushReason.ToProto(),
315+
ContentBytesLoaded: s.ContentBytesLoaded,
316+
IndexBytesLoaded: s.IndexBytesLoaded,
317+
Crashes: int64(s.Crashes),
318+
Duration: durationpb.New(s.Duration),
319+
FileCount: int64(s.FileCount),
320+
ShardFilesConsidered: int64(s.ShardFilesConsidered),
321+
FilesConsidered: int64(s.FilesConsidered),
322+
FilesLoaded: int64(s.FilesLoaded),
323+
FilesSkipped: int64(s.FilesSkipped),
324+
FilesSkippedDueToCancellation: int64(s.FilesSkippedDueToCancellation),
325+
ShardsScanned: int64(s.ShardsScanned),
326+
ShardsSkipped: int64(s.ShardsSkipped),
327+
ShardsSkippedFilter: int64(s.ShardsSkippedFilter),
328+
MatchCount: int64(s.MatchCount),
329+
NgramMatches: int64(s.NgramMatches),
330+
NgramLookups: int64(s.NgramLookups),
331+
Wait: durationpb.New(s.Wait),
332+
MatchTreeConstruction: durationpb.New(s.MatchTreeConstruction),
333+
MatchTreeSearch: durationpb.New(s.MatchTreeSearch),
334+
RegexpsConsidered: int64(s.RegexpsConsidered),
335+
FlushReason: s.FlushReason.ToProto(),
334336
}
335337
}
336338

‎api_test.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,7 @@ func benchmarkEncoding(data any) func(*testing.B) {
8787

8888
func TestSizeBytesSearchResult(t *testing.T) {
8989
sr := SearchResult{
90-
Stats: Stats{}, // 129 bytes
90+
Stats: Stats{}, // 137 bytes
9191
Progress: Progress{}, // 16 bytes
9292
Files: []FileMatch{{ // 24 bytes + 460 bytes
9393
Score: 0, // 8 bytes
@@ -118,7 +118,7 @@ func TestSizeBytesSearchResult(t *testing.T) {
118118
LineFragments: nil, // 48 bytes
119119
}
120120

121-
var wantBytes uint64 = 725
121+
var wantBytes uint64 = 733
122122
if sr.SizeBytes() != wantBytes {
123123
t.Fatalf("want %d, got %d", wantBytes, sr.SizeBytes())
124124
}

‎cmd/zoekt-webserver/main.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -563,6 +563,7 @@ func (s *loggedSearcher) log(ctx context.Context, q query.Q, opts *zoekt.SearchO
563563
sglog.Int("stat.FilesConsidered", st.FilesConsidered),
564564
sglog.Int("stat.FilesLoaded", st.FilesLoaded),
565565
sglog.Int("stat.FilesSkipped", st.FilesSkipped),
566+
sglog.Int("stat.FilesSkippedDueToCancellation", st.FilesSkippedDueToCancellation),
566567
sglog.Int("stat.ShardsScanned", st.ShardsScanned),
567568
sglog.Int("stat.ShardsSkipped", st.ShardsSkipped),
568569
sglog.Int("stat.ShardsSkippedFilter", st.ShardsSkippedFilter),

0 commit comments

Comments
 (0)