Facebook sync (May 2019) + relicense (#838)

Changelog:

- changed license: BSD+Patents -> MIT
- propagates exceptions raised in sub-indexes of IndexShards and IndexReplicas
- support for searching several inverted lists in parallel (parallel_mode != 0)
- better support for PQ codes where nbit != 8 or 16
- IVFSpectralHash implementation: spectral hash codes inside an IVF
- 6-bit per component scalar quantizer (4 and 8 bit were already supported)
- combinations of inverted lists: HStackInvertedLists and VStackInvertedLists
- configurable number of threads for OnDiskInvertedLists prefetching (including 0=no prefetch)
- more test and demo code compatible with Python 3 (print with parentheses)
- refactored benchmark code: data loading is now in a single file

Author: beauby

AutoTune.cpp

@@ -1,8 +1,7 @@
 /**
- * Copyright (c) 2015-present, Facebook, Inc.
- * All rights reserved.
+ * Copyright (c) Facebook, Inc. and its affiliates.
  *
- * This source code is licensed under the BSD+Patents license found in the
+ * This source code is licensed under the MIT license found in the
  * LICENSE file in the root directory of this source tree.
  */
 
@@ -469,9 +468,22 @@ void ParameterSpace::set_index_parameter (
     }
     if (DC (IndexShards)) {
         // call on all sub-indexes
-        for (auto & shard_index : ix->shard_indexes) {
-            set_index_parameter (shard_index, name, val);
-        }
+        auto fn =
+          [this, name, val](int, Index* subIndex) {
+            set_index_parameter(subIndex, name, val);
+          };
+
+        ix->runOnIndex(fn);
+        return;
+    }
+    if (DC (IndexReplicas)) {
+        // call on all sub-indexes
+        auto fn =
+          [this, name, val](int, Index* subIndex) {
+            set_index_parameter(subIndex, name, val);
+          };
+
+        ix->runOnIndex(fn);
         return;
     }
     if (DC (IndexRefineFlat)) {

AutoTune.h

@@ -1,8 +1,7 @@
 /**
- * Copyright (c) 2015-present, Facebook, Inc.
- * All rights reserved.
+ * Copyright (c) Facebook, Inc. and its affiliates.
  *
- * This source code is licensed under the BSD+Patents license found in the
+ * This source code is licensed under the MIT license found in the
  * LICENSE file in the root directory of this source tree.
  */
 

AuxIndexStructures.cpp

@@ -1,8 +1,7 @@
 /**
- * Copyright (c) 2015-present, Facebook, Inc.
- * All rights reserved.
+ * Copyright (c) Facebook, Inc. and its affiliates.
  *
- * This source code is licensed under the BSD+Patents license found in the
+ * This source code is licensed under the MIT license found in the
  * LICENSE file in the root directory of this source tree.
  */
 
@@ -54,6 +53,10 @@ RangeSearchResult::~RangeSearchResult () {
     delete [] lims;
 }
 
+
+
+
+
 /***********************************************************************
  * BufferList
  ***********************************************************************/
@@ -148,7 +151,7 @@ void RangeSearchPartialResult::finalize ()
     res->do_allocation ();
 
 #pragma omp barrier
-    set_result ();
+    copy_result ();
 }
 
 
@@ -162,7 +165,7 @@ void RangeSearchPartialResult::set_lims ()
 }
 
 /// called by range_search after do_allocation
-void RangeSearchPartialResult::set_result (bool incremental)
+void RangeSearchPartialResult::copy_result (bool incremental)
 {
     size_t ofs = 0;
     for (int i = 0; i < queries.size(); i++) {
@@ -178,6 +181,38 @@ void RangeSearchPartialResult::set_result (bool incremental)
     }
 }
 
+void RangeSearchPartialResult::merge (std::vector <RangeSearchPartialResult *> &
+                                      partial_results, bool do_delete)
+{
+
+    int npres = partial_results.size();
+    if (npres == 0) return;
+    RangeSearchResult *result = partial_results[0]->res;
+    size_t nx = result->nq;
+
+    // count
+    for (const RangeSearchPartialResult * pres : partial_results) {
+        if (!pres) continue;
+        for (const RangeQueryResult &qres : pres->queries) {
+            result->lims[qres.qno] += qres.nres;
+        }
+    }
+    result->do_allocation ();
+    for (int j = 0; j < npres; j++) {
+        if (!partial_results[j]) continue;
+        partial_results[j]->copy_result (true);
+        if (do_delete) {
+            delete partial_results[j];
+            partial_results[j] = nullptr;
+        }
+    }
+
+    // reset the limits
+    for (size_t i = nx; i > 0; i--) {
+        result->lims [i] = result->lims [i - 1];
+    }
+    result->lims [0] = 0;
+}
 
 /***********************************************************************
  * IDSelectorRange

AuxIndexStructures.h

@@ -1,8 +1,7 @@
 /**
- * Copyright (c) 2015-present, Facebook, Inc.
- * All rights reserved.
+ * Copyright (c) Facebook, Inc. and its affiliates.
  *
- * This source code is licensed under the BSD+Patents license found in the
+ * This source code is licensed under the MIT license found in the
  * LICENSE file in the root directory of this source tree.
  */
 
@@ -90,10 +89,15 @@ struct IDSelectorBatch: IDSelector {
     ~IDSelectorBatch() override {}
 };
 
-
-// Below are structures used only by Index implementations
-
-
+/****************************************************************
+ * Result structures for range search.
+ *
+ * The main constraint here is that we want to support parallel
+ * queries from different threads in various ways: 1 thread per query,
+ * several threads per query. We store the actual results in blocks of
+ * fixed size rather than exponentially increasing memory. At the end,
+ * we copy the block content to a linear result array.
+ *****************************************************************/
 
 /** List of temporary buffers used to store results before they are
  *  copied to the RangeSearchResult object. */
@@ -115,9 +119,10 @@ struct BufferList {
 
     ~BufferList ();
 
-    // create a new buffer
+    /// create a new buffer
     void append_buffer ();
 
+    /// add one result, possibly appending a new buffer if needed
     void add (idx_t id, float dis);
 
     /// copy elemnts ofs:ofs+n-1 seen as linear data in the buffers to
@@ -132,31 +137,42 @@ struct RangeSearchPartialResult;
 /// result structure for a single query
 struct RangeQueryResult {
     using idx_t = Index::idx_t;
-    idx_t qno;
-    size_t nres;
+    idx_t qno;    //< id of the query
+    size_t nres;  //< nb of results for this query
     RangeSearchPartialResult * pres;
 
+    /// called by search function to report a new result
     void add (float dis, idx_t id);
 };
 
 /// the entries in the buffers are split per query
 struct RangeSearchPartialResult: BufferList {
     RangeSearchResult * res;
 
+    /// eventually the result will be stored in res_in
     explicit RangeSearchPartialResult (RangeSearchResult * res_in);
 
+    /// query ids + nb of results per query.
     std::vector<RangeQueryResult> queries;
 
     /// begin a new result
     RangeQueryResult & new_result (idx_t qno);
 
+    /*****************************************
+     * functions used at the end of the search to merge the result
+     * lists */
     void finalize ();
 
     /// called by range_search before do_allocation
     void set_lims ();
 
     /// called by range_search after do_allocation
-    void set_result (bool incremental = false);
+    void copy_result (bool incremental = false);
+
+    /// merge a set of PartialResult's into one RangeSearchResult
+    /// on ouptut the partialresults are empty!
+    static void merge (std::vector <RangeSearchPartialResult *> &
+                       partial_results, bool do_delete=true);
 
 };
 
@@ -212,7 +228,7 @@ struct VectorIOWriter:IOWriter {
  * it maintains counters) so the distance functions are not const,
  * instanciate one from each thread if needed.
  ***********************************************************/
- struct DistanceComputer {
+struct DistanceComputer {
      using idx_t = Index::idx_t;
 
      /// called before computing distances
@@ -225,7 +241,7 @@ struct VectorIOWriter:IOWriter {
      virtual float symmetric_dis (idx_t i, idx_t j) = 0;
 
      virtual ~DistanceComputer() {}
- };
+};
 
 /***********************************************************
  * Interrupt callback

Clustering.cpp

@@ -1,8 +1,7 @@
 /**
- * Copyright (c) 2015-present, Facebook, Inc.
- * All rights reserved.
+ * Copyright (c) Facebook, Inc. and its affiliates.
  *
- * This source code is licensed under the BSD+Patents license found in the
+ * This source code is licensed under the MIT license found in the
  * LICENSE file in the root directory of this source tree.
  */
 

Clustering.h

@@ -1,8 +1,7 @@
 /**
- * Copyright (c) 2015-present, Facebook, Inc.
- * All rights reserved.
+ * Copyright (c) Facebook, Inc. and its affiliates.
  *
- * This source code is licensed under the BSD+Patents license found in the
+ * This source code is licensed under the MIT license found in the
  * LICENSE file in the root directory of this source tree.
  */
 

FaissAssert.h

@@ -1,8 +1,7 @@
 /**
- * Copyright (c) 2015-present, Facebook, Inc.
- * All rights reserved.
+ * Copyright (c) Facebook, Inc. and its affiliates.
  *
- * This source code is licensed under the BSD+Patents license found in the
+ * This source code is licensed under the MIT license found in the
  * LICENSE file in the root directory of this source tree.
  */
 

FaissException.cpp

@@ -1,14 +1,14 @@
 /**
- * Copyright (c) 2015-present, Facebook, Inc.
- * All rights reserved.
+ * Copyright (c) Facebook, Inc. and its affiliates.
  *
- * This source code is licensed under the BSD+Patents license found in the
+ * This source code is licensed under the MIT license found in the
  * LICENSE file in the root directory of this source tree.
  */
 
 // -*- c++ -*-
 
 #include "FaissException.h"
+#include <sstream>
 
 namespace faiss {
 
@@ -32,4 +32,35 @@ FaissException::what() const noexcept {
   return msg.c_str();
 }
 
+void handleExceptions(
+  std::vector<std::pair<int, std::exception_ptr>>& exceptions) {
+  if (exceptions.size() == 1) {
+    // throw the single received exception directly
+    std::rethrow_exception(exceptions.front().second);
+
+  } else if (exceptions.size() > 1) {
+    // multiple exceptions; aggregate them and return a single exception
+    std::stringstream ss;
+
+    for (auto& p : exceptions) {
+      try {
+        std::rethrow_exception(p.second);
+      } catch (std::exception& ex) {
+        if (ex.what()) {
+          // exception message available
+          ss << "Exception thrown from index " << p.first << ": "
+             << ex.what() << "\n";
+        } else {
+          // No message available
+          ss << "Unknown exception thrown from index " << p.first << "\n";
+        }
+      } catch (...) {
+        ss << "Unknown exception thrown from index " << p.first << "\n";
+      }
+    }
+
+    throw FaissException(ss.str());
+  }
+}
+
 }

FaissException.h

@@ -1,8 +1,7 @@
 /**
- * Copyright (c) 2015-present, Facebook, Inc.
- * All rights reserved.
+ * Copyright (c) Facebook, Inc. and its affiliates.
  *
- * This source code is licensed under the BSD+Patents license found in the
+ * This source code is licensed under the MIT license found in the
  * LICENSE file in the root directory of this source tree.
  */
 
@@ -13,6 +12,8 @@
 
 #include <exception>
 #include <string>
+#include <vector>
+#include <utility>
 
 namespace faiss {
 
@@ -32,6 +33,11 @@ class FaissException : public std::exception {
   std::string msg;
 };
 
+/// Handle multiple exceptions from worker threads, throwing an appropriate
+/// exception that aggregates the information
+/// The pair int is the thread that generated the exception
+void
+handleExceptions(std::vector<std::pair<int, std::exception_ptr>>& exceptions);
 
 /** bare-bones unique_ptr
  * this one deletes with delete [] */
@@ -60,9 +66,6 @@ struct ScopeDeleter1 {
     }
 };
 
-
-
 }
 
-
 #endif

HNSW.cpp

@@ -1,8 +1,7 @@
 /**
- * Copyright (c) 2015-present, Facebook, Inc.
- * All rights reserved.
+ * Copyright (c) Facebook, Inc. and its affiliates.
  *
- * This source code is licensed under the BSD+Patents license found in the
+ * This source code is licensed under the MIT license found in the
  * LICENSE file in the root directory of this source tree.
  */