Additional input validation for index deserialization (#4899)

Summary:
Pull Request resolved: #4899

Add bounds checks when reading index data from untrusted
byte streams. Five new FAISS_THROW_IF_NOT_FMT guards reject
invalid values early during deserialization:

- ProductAdditiveQuantizer: nsplits must be > 0
- ScalarQuantizer: qtype must be within the valid
  QuantizerType enum range
- NSG: R (max out-degree) must be > 0
- IndexPreTransform: VectorTransform chain length must
  be >= 0
- IndexBinaryMultiHash: nhash must be > 0

Each check includes a descriptive error message with the
offending value. Without these checks, invalid data could
cause undefined behavior such as zero-size allocations,
out-of-range enum casts, or negative loop bounds.

Reviewed By: mnorris11

Differential Revision: D95968069

fbshipit-source-id: b9a3e88a01d3427614fa0e027fb233f11ee8cf2e

Author: scsiguy

faiss/impl/index_read.cpp

@@ -553,6 +553,10 @@ static void read_ProductAdditiveQuantizer(
         IOReader* f) {
     read_AdditiveQuantizer(paq, f);
     READ1(paq.nsplits);
+    FAISS_THROW_IF_NOT_FMT(
+            paq.nsplits > 0,
+            "invalid ProductAdditiveQuantizer nsplits %zd (must be > 0)",
+            paq.nsplits);
 }
 
 static void read_ProductResidualQuantizer(
@@ -581,7 +585,14 @@ static void read_ProductLocalSearchQuantizer(
 }
 
 void read_ScalarQuantizer(ScalarQuantizer* ivsc, IOReader* f) {
-    READ1(ivsc->qtype);
+    int qtype_int;
+    READ1(qtype_int);
+    FAISS_THROW_IF_NOT_FMT(
+            qtype_int >= ScalarQuantizer::QT_8bit &&
+                    qtype_int <= ScalarQuantizer::QT_8bit_direct_signed,
+            "invalid ScalarQuantizer qtype %d",
+            qtype_int);
+    ivsc->qtype = static_cast<ScalarQuantizer::QuantizerType>(qtype_int);
     READ1(ivsc->rangestat);
     READ1(ivsc->rangestat_arg);
     READ1(ivsc->d);
@@ -724,6 +735,7 @@ static void read_HNSW(HNSW& hnsw, IOReader* f) {
 static void read_NSG(NSG& nsg, IOReader* f) {
     READ1(nsg.ntotal);
     READ1(nsg.R);
+    FAISS_THROW_IF_NOT_FMT(nsg.R > 0, "invalid NSG R %d (must be > 0)", nsg.R);
     READ1(nsg.L);
     READ1(nsg.C);
     READ1(nsg.search_L);
@@ -1352,6 +1364,10 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
         } else {
             READ1(nt);
         }
+        FAISS_THROW_IF_NOT_FMT(
+                nt >= 0,
+                "invalid VectorTransform chain length %d (must be >= 0)",
+                nt);
         for (int i = 0; i < nt; i++) {
             ixpt->chain.push_back(read_VectorTransform(f));
         }
@@ -2043,6 +2059,10 @@ std::unique_ptr<IndexBinary> read_index_binary_up(IOReader* f, int io_flags) {
         idxmh->own_fields = true;
         READ1(idxmh->b);
         READ1(idxmh->nhash);
+        FAISS_THROW_IF_NOT_FMT(
+                idxmh->nhash > 0,
+                "invalid IndexBinaryMultiHash nhash %d (must be > 0)",
+                idxmh->nhash);
         READ1(idxmh->nflip);
         idxmh->maps.resize(idxmh->nhash);
         for (int i = 0; i < idxmh->nhash; i++) {

tests/test_read_index_deserialize.cpp

@@ -440,6 +440,95 @@ TEST(ReadIndexDeserialize, BinaryHashEmptyInvlistBuffer) {
     expect_binary_read_throws_with(buf, "binary hash invlists");
 }
 
+// -----------------------------------------------------------------------
+// Test: NSG with R=0 triggers the R > 0 validation.
+// -----------------------------------------------------------------------
+TEST(ReadIndexDeserialize, NSGNegativeR) {
+    // "INSf" format: fourcc + index_header + GK + build_type +
+    //   nndescent_S/R/L/iter + read_NSG(ntotal, R, ...)
+    std::vector<uint8_t> buf;
+    push_fourcc(buf, "INSf");
+    push_index_header(buf, /*d=*/4, /*ntotal=*/0);
+    push_val<int>(buf, 0);  // GK
+    push_val<int>(buf, 0);  // build_type
+    push_val<int>(buf, 10); // nndescent_S
+    push_val<int>(buf, 10); // nndescent_R
+    push_val<int>(buf, 10); // nndescent_L
+    push_val<int>(buf, 1);  // nndescent_iter
+    // read_NSG fields:
+    push_val<int>(buf, 0);  // ntotal
+    push_val<int>(buf, -1); // R = -1 (invalid)
+
+    expect_read_throws_with(buf, "invalid NSG R");
+}
+
+// -----------------------------------------------------------------------
+// Test: ScalarQuantizer with out-of-range qtype throws.
+// -----------------------------------------------------------------------
+TEST(ReadIndexDeserialize, ScalarQuantizerInvalidQtype) {
+    // "IxSQ" format: fourcc + index_header + read_ScalarQuantizer(qtype, ...)
+    std::vector<uint8_t> buf;
+    push_fourcc(buf, "IxSQ");
+    push_index_header(buf, /*d=*/4, /*ntotal=*/0);
+    // ScalarQuantizer fields:
+    push_val<int>(buf, 99); // qtype = 99 (out of range)
+
+    expect_read_throws_with(buf, "qtype");
+}
+
+// -----------------------------------------------------------------------
+// Test: ProductAdditiveQuantizer with nsplits=0 throws.
+// -----------------------------------------------------------------------
+TEST(ReadIndexDeserialize, ProductAdditiveQuantizerZeroNsplits) {
+    // "IxPR" format: fourcc + index_header +
+    //   read_ProductResidualQuantizer(read_ProductAdditiveQuantizer(
+    //     read_AdditiveQuantizer(...) + nsplits))
+    std::vector<uint8_t> buf;
+    push_fourcc(buf, "IxPR");
+    push_index_header(buf, /*d=*/4, /*ntotal=*/0);
+    // AdditiveQuantizer fields:
+    push_val<size_t>(buf, 4);      // d
+    push_val<size_t>(buf, 1);      // M
+    push_vector<size_t>(buf, {8}); // nbits (1 element matching M=1)
+    push_val<bool>(buf, true);     // is_trained
+    push_vector<float>(buf, {});   // codebooks (empty)
+    push_val<int>(buf, 0);         // search_type = ST_decompress
+    push_val<float>(buf, 0.0f);    // norm_min
+    push_val<float>(buf, 1.0f);    // norm_max
+    // ProductAdditiveQuantizer field:
+    push_val<size_t>(buf, 0); // nsplits = 0 (invalid)
+
+    expect_read_throws_with(buf, "nsplits");
+}
+
+// -----------------------------------------------------------------------
+// Test: PreTransform with negative chain length throws.
+// -----------------------------------------------------------------------
+TEST(ReadIndexDeserialize, PreTransformNegativeChainLength) {
+    // "IxPT" format: fourcc + index_header + nt + VT chain + nested index
+    std::vector<uint8_t> buf;
+    push_fourcc(buf, "IxPT");
+    push_index_header(buf, /*d=*/4, /*ntotal=*/0);
+    push_val<int>(buf, -1); // nt = -1 (invalid)
+
+    expect_read_throws_with(buf, "chain length");
+}
+
+// -----------------------------------------------------------------------
+// Test: IndexBinaryMultiHash with nhash=0 throws.
+// -----------------------------------------------------------------------
+TEST(ReadIndexDeserialize, BinaryMultiHashZeroNhash) {
+    std::vector<uint8_t> buf;
+    push_fourcc(buf, "IBHm");
+    push_binary_index_header(buf, /*d=*/16, /*ntotal=*/0);
+    // Nested IBxF storage (ntotal=0 matches outer)
+    push_minimal_binary_flat(buf, /*d=*/16);
+    push_val<int>(buf, 4); // b
+    push_val<int>(buf, 0); // nhash = 0 (invalid)
+
+    expect_binary_read_throws_with(buf, "nhash");
+}
+
 // -----------------------------------------------------------------------
 // Test: IndexBinaryHash with b=0 triggers the b > 0 validation.
 // Without this check, BitstringReader::read(0) would silently produce