Skip to content

Validate code_size against compute_code_size in RaBitQ flat deserialization #5432

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
AlexMelanFromRingo wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Validate code_size against compute_code_size in RaBitQ flat deserialization #5432

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions faiss/impl/index_read.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2715,6 +2715,11 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {

// rabitq.nb_bits is already set to 1 by read_RaBitQuantizer
idxq->code_size = idxq->rabitq.code_size;
validate_code_size_match(
idxq->code_size,
idxq->rabitq.compute_code_size(
idxq->rabitq.d, idxq->rabitq.nb_bits),
"IndexRaBitQ");
idx = std::move(idxq);
} else if (h == fourcc("Ixrr")) {
// Ixrr = multi-bit format (new)
Expand All @@ -2733,6 +2738,11 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
idxq->qb);

idxq->code_size = idxq->rabitq.code_size;
validate_code_size_match(
idxq->code_size,
idxq->rabitq.compute_code_size(
idxq->rabitq.d, idxq->rabitq.nb_bits),
"IndexRaBitQ");
idx = std::move(idxq);
} else if (h == fourcc("Iwrq")) {
auto ivrq = std::make_unique<IndexIVFRaBitQ>();
Expand Down
59 changes: 52 additions & 7 deletions tests/test_read_index_deserialize.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2175,19 +2175,24 @@ TEST(ReadIndexDeserialize, ITQTransformMeanTooSmall) {
// RaBitQ qb deserialization validation tests
// -----------------------------------------------------------------------

/// Single-bit RaBitQ code_size == compute_code_size(d, 1): (d+7)/8 + 8.
static size_t rabitq_single_bit_code_size(size_t d) {
return (d + 7) / 8 + 8;
}

/// Helper: push a minimal RaBitQuantizer (single-bit format, multi_bit=false).
static void push_rabitq(std::vector<uint8_t>& buf, size_t d) {
push_val<size_t>(buf, d); // d
push_val<size_t>(buf, 1); // code_size
push_val<int>(buf, 1); // metric_type (L2)
push_val<size_t>(buf, d); // d
push_val<size_t>(buf, rabitq_single_bit_code_size(d)); // code_size
push_val<int>(buf, 1); // metric_type (L2)
}

/// Helper: push a minimal RaBitQuantizer (multi-bit format, multi_bit=true).
static void push_rabitq_multibit(std::vector<uint8_t>& buf, size_t d) {
push_val<size_t>(buf, d); // d
push_val<size_t>(buf, 1); // code_size
push_val<int>(buf, 1); // metric_type (L2)
push_val<size_t>(buf, 1); // nb_bits
push_val<size_t>(buf, d); // d
push_val<size_t>(buf, rabitq_single_bit_code_size(d)); // code_size (nb_bits=1)
push_val<int>(buf, 1); // metric_type (L2)
push_val<size_t>(buf, 1); // nb_bits
}

/// Helper: push an IVF header (index_header + nlist + nprobe + flat quantizer
Expand Down Expand Up @@ -2287,6 +2292,46 @@ TEST(ReadIndexDeserialize, RaBitQQbZeroAccepted_Ixrr) {
EXPECT_NO_THROW(read_index_up(&reader));
}

// -----------------------------------------------------------------------
// Test: IndexRaBitQ code_size field mismatch. The flat readers take
// rabitq.code_size straight from the file. If it does not match
// compute_code_size(d, nb_bits), decode_core still reads the per-code
// factor block at offset (d+7)/8 -- derived from d, not code_size -- so a
// too-small code_size makes search()/sa_decode() read past the codes
// buffer (heap OOB), even when codes.size() == ntotal * code_size holds.
// -----------------------------------------------------------------------
TEST(ReadIndexDeserialize, RaBitQCodeSizeFieldMismatch_Ixrq) {
std::vector<uint8_t> buf;
push_fourcc(buf, "Ixrq");
push_index_header(buf, /*d=*/8, /*ntotal=*/0);
// RaBitQuantizer with a code_size that does not match
// compute_code_size(8, 1) == (8+7)/8 + 8 == 9.
push_val<size_t>(buf, (size_t)8); // d
push_val<size_t>(buf, (size_t)1); // code_size = 1 (forged; real = 9)
push_val<int>(buf, 1); // metric_type (L2)
push_vector<uint8_t>(buf, {}); // codes
push_vector<float>(buf, std::vector<float>(8, 0.0f)); // center
push_val<uint8_t>(buf, 0); // qb = 0

expect_read_throws_with(buf, "code_size mismatch");
}

TEST(ReadIndexDeserialize, RaBitQCodeSizeFieldMismatch_Ixrr) {
std::vector<uint8_t> buf;
push_fourcc(buf, "Ixrr");
push_index_header(buf, /*d=*/8, /*ntotal=*/0);
// Multi-bit form: code_size must match compute_code_size(8, nb_bits).
push_val<size_t>(buf, (size_t)8); // d
push_val<size_t>(buf, (size_t)1); // code_size = 1 (forged; real = 9 for nb_bits=1)
push_val<int>(buf, 1); // metric_type (L2)
push_val<size_t>(buf, 1); // nb_bits = 1
push_vector<uint8_t>(buf, {}); // codes
push_vector<float>(buf, std::vector<float>(8, 0.0f)); // center
push_val<uint8_t>(buf, 0); // qb = 0

expect_read_throws_with(buf, "code_size mismatch");
}

// -- Irfn (IndexRaBitQFastScan, new format) --
// qb=0 is not supported: FastScan requires quantized queries for SIMD.

Expand Down
Loading