index: report malformed posting lists as shard corruption

index/hititer.go

@@ -187,6 +187,7 @@ type compressedPostingIterator struct {
 
 func newCompressedPostingIterator(b []byte, w ngram) *compressedPostingIterator {
 	d, sz := binary.Uvarint(b)
+	validatePostingVarint(b, sz)
 	return &compressedPostingIterator{
 		_first:           uint32(d),
 		blob:             b[sz:],
@@ -212,6 +213,7 @@ func (i *compressedPostingIterator) next(limit uint32) {
 
 	for i._first <= limit && len(i.blob) > 0 {
 		delta, sz := binary.Uvarint(i.blob)
+		validatePostingVarint(i.blob, sz)
 		i._first += uint32(delta)
 		i.indexBytesLoaded += sz
 		i.blob = i.blob[sz:]
@@ -222,6 +224,12 @@ func (i *compressedPostingIterator) next(limit uint32) {
 	}
 }
 
+// validatePostingVarint intentionally relies on the bounds check to panic for
+// the non-positive lengths binary.Uvarint returns for malformed input.
+func validatePostingVarint(blob []byte, sz int) {
+	_ = blob[sz-1]
+}
+
 func (i *compressedPostingIterator) updateStats(s *zoekt.Stats) {
 	s.IndexBytesLoaded += int64(i.indexBytesLoaded)
 	s.NgramLookups += i.ngramLookups

index/hititer_test.go

@@ -112,3 +112,41 @@ func genUints32(size int) []uint32 {
 	}
 	return nums
 }
+
+func TestCompressedPostingIteratorMalformedVarint(t *testing.T) {
+	overflow := []byte{0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x01}
+
+	for _, tc := range []struct {
+		name string
+		bad  []byte
+	}{
+		{name: "truncated", bad: []byte{0x80}},
+		{name: "overflowing", bad: overflow},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Run("first", func(t *testing.T) {
+				assertPanic(t, func() {
+					newCompressedPostingIterator(tc.bad, stringToNGram("abc"))
+				})
+			})
+
+			t.Run("delta", func(t *testing.T) {
+				blob := append([]byte{0x01}, tc.bad...)
+				it := newCompressedPostingIterator(blob, stringToNGram("abc"))
+				assertPanic(t, func() {
+					it.next(100)
+				})
+			})
+		})
+	}
+}
+
+func assertPanic(t *testing.T, f func()) {
+	t.Helper()
+	defer func() {
+		if got := recover(); got == nil {
+			t.Fatal("got no panic, want corruption panic")
+		}
+	}()
+	f()
+}