Reject null quantizer during IVF index deserialization (facebookresearch#5112)

Summary:

Add validation in read_ivf_header() to reject a null quantizer sub-index read from serialized data. The IVF deserialization reads the quantizer via read_index(), which returns nullptr when the stream contains the "null" fourcc. A null quantizer is fundamentally invalid for any IVF index type. Without this check, downstream code (e.g. initialize_IVFPQ_precomputed_table, IndexIVF::search) dereferences the null pointer.

This single validation protects all IVF index types that share read_ivf_header: IndexIVFFlat, IndexIVFPQ, IndexIVFScalarQuantizer, IndexIVFAdditiveQuantizer, and others.

Reviewed By: mnorris11

Differential Revision: D101236489

Author: scsiguy

faiss/IndexIVF.cpp

@@ -58,6 +58,7 @@ void Level1Quantizer::train_q1(
         const float* x,
         bool verbose,
         MetricType metric_type) {
+    FAISS_THROW_IF_NOT_MSG(quantizer, "IVF quantizer must not be null");
     size_t d = quantizer->d;
     if (quantizer->is_trained &&
         (static_cast<size_t>(quantizer->ntotal) == nlist)) {
@@ -188,6 +189,7 @@ void IndexIVF::add(idx_t n, const float* x) {
 }
 
 void IndexIVF::add_with_ids(idx_t n, const float* x, const idx_t* xids) {
+    FAISS_THROW_IF_NOT_MSG(quantizer, "IVF quantizer must not be null");
     FAISS_THROW_IF_NOT_MSG(invlists, "IVF index has no inverted lists");
     std::unique_ptr<idx_t[]> coarse_idx(new idx_t[n]);
     quantizer->assign(n, x, coarse_idx.get());
@@ -309,6 +311,7 @@ void IndexIVF::search(
         idx_t* labels,
         const SearchParameters* params_in) const {
     FAISS_THROW_IF_NOT(k > 0);
+    FAISS_THROW_IF_NOT_MSG(quantizer, "IVF quantizer must not be null");
     FAISS_THROW_IF_NOT_MSG(invlists, "IVF index has no inverted lists");
     const IVFSearchParameters* params = nullptr;
     if (params_in) {
@@ -734,6 +737,7 @@ void IndexIVF::range_search(
         float radius,
         RangeSearchResult* result,
         const SearchParameters* params_in) const {
+    FAISS_THROW_IF_NOT_MSG(quantizer, "IVF quantizer must not be null");
     const IVFSearchParameters* params = nullptr;
     const SearchParameters* quantizer_params = nullptr;
     if (params_in) {

faiss/IndexIVFPQ.cpp

@@ -388,6 +388,7 @@ void initialize_IVFPQ_precomputed_table(
         AlignedTable<float>& precomputed_table,
         bool by_residual,
         bool verbose) {
+    FAISS_THROW_IF_NOT_MSG(quantizer, "IVF quantizer must not be null");
     size_t nlist = quantizer->ntotal;
     size_t d = quantizer->d;
     FAISS_THROW_IF_NOT(d == pq.d);

tests/test_read_index_deserialize.cpp

@@ -22,6 +22,7 @@
 #include <faiss/IndexIVFAdditiveQuantizerFastScan.h>
 #include <faiss/IndexIVFFlat.h>
 #include <faiss/IndexIVFIndependentQuantizer.h>
+#include <faiss/IndexIVFPQ.h>
 #include <faiss/IndexIVFPQR.h>
 #include <faiss/IndexRaBitQFastScan.h>
 #include <faiss/VectorTransform.h>
@@ -1647,6 +1648,62 @@ TEST(ReadIndexDeserialize, IVFQuantizerUntrained) {
     EXPECT_NO_THROW(read_index_up(&reader));
 }
 
+// -----------------------------------------------------------------------
+// Test: initialize_IVFPQ_precomputed_table rejects a null quantizer.
+// Protects against null-deref from corrupt serialized data where the
+// quantizer sub-index is absent (fourcc "null").
+// -----------------------------------------------------------------------
+TEST(ReadIndexDeserialize, IVFPQNullQuantizerPrecomputeTableRejected) {
+    ProductQuantizer pq(4, 1, 8);
+    AlignedTable<float> precomputed_table;
+    int use_precomputed_table = 0;
+    EXPECT_THROW(
+            initialize_IVFPQ_precomputed_table(
+                    use_precomputed_table,
+                    /*quantizer=*/nullptr,
+                    pq,
+                    precomputed_table,
+                    /*by_residual=*/true,
+                    /*verbose=*/false),
+            faiss::FaissException);
+}
+
+TEST(ReadIndexDeserialize, IVFNullQuantizerSearchRejected) {
+    IndexIVFFlat ivf;
+    ivf.quantizer = nullptr;
+    ivf.is_trained = true;
+    std::vector<float> x(4);
+    std::vector<float> distances(1);
+    std::vector<idx_t> labels(1);
+    EXPECT_THROW(
+            ivf.search(1, x.data(), 1, distances.data(), labels.data()),
+            faiss::FaissException);
+}
+
+TEST(ReadIndexDeserialize, IVFNullQuantizerRangeSearchRejected) {
+    IndexIVFFlat ivf;
+    ivf.quantizer = nullptr;
+    ivf.is_trained = true;
+    std::vector<float> x(4);
+    RangeSearchResult result(1);
+    EXPECT_THROW(
+            ivf.range_search(1, x.data(), 1.0, &result), faiss::FaissException);
+}
+
+TEST(ReadIndexDeserialize, IVFNullQuantizerAddRejected) {
+    IndexIVFFlat ivf;
+    ivf.quantizer = nullptr;
+    std::vector<float> x(4);
+    EXPECT_THROW(ivf.add(1, x.data()), faiss::FaissException);
+}
+
+TEST(ReadIndexDeserialize, IVFNullQuantizerTrainRejected) {
+    IndexIVFFlat ivf;
+    ivf.quantizer = nullptr;
+    std::vector<float> x(4);
+    EXPECT_THROW(ivf.train(1, x.data()), faiss::FaissException);
+}
+
 // -----------------------------------------------------------------------
 // VectorTransform deserialization validation tests
 // -----------------------------------------------------------------------