Merge shard results by metric type rather than assuming similarity

faiss/IndexShards.cpp

@@ -243,8 +243,11 @@ void IndexShardsTemplate<IndexT>::search(
 
     this->runOnIndex(fn);
 
-    if (this->metric_type == METRIC_L2) {
-        merge_knn_results<idx_t, CMin<distance_t, int>>(
+    // Only similarity metrics rank larger values first. Testing for METRIC_L2 alone merged every
+    // other metric as a similarity, so distance metrics such as METRIC_L1, METRIC_Linf and METRIC_Lp
+    // came back worst-first once results crossed shard boundaries.
+    if (is_similarity_metric(this->metric_type)) {
+        merge_knn_results<idx_t, CMax<distance_t, int>>(
                 n,
                 k,
                 nshard,
@@ -253,7 +256,7 @@ void IndexShardsTemplate<IndexT>::search(
                 distances,
                 labels);
     } else {
-        merge_knn_results<idx_t, CMax<distance_t, int>>(
+        merge_knn_results<idx_t, CMin<distance_t, int>>(
                 n,
                 k,
                 nshard,

tests/test_meta_index.py

@@ -145,6 +145,34 @@ def test_shards(self):
             # thousands of the nq*k cells, far above this floor.
             assert ndiff < nq * k / 100.0, f"too many mismatches: {ndiff}"
 
+    def test_shards_distance_metric_ordering(self):
+        # METRIC_L1 returns distances, so the shard merge must rank smaller values first.
+        # Testing only for METRIC_L2 treated every other metric as a similarity, which put the
+        # FARTHEST vectors first as soon as results crossed a shard boundary.
+        k = 10
+        ref_index = faiss.IndexFlat(d, faiss.METRIC_L1)
+        ref_index.add(xb)
+        Dref, _Iref = ref_index.search(xq, k)
+
+        shard_index = faiss.IndexShards(d, False, True)
+        shards = []
+        ni = 3
+        for i in range(ni):
+            i0 = int(i * nb / ni)
+            i1 = int((i + 1) * nb / ni)
+            shard = faiss.IndexFlat(d, faiss.METRIC_L1)
+            shard.add(xb[i0:i1])
+            shards.append(shard)  # keep the shards alive for the duration of the test
+            shard_index.add_shard(shard)
+
+        D, _I = shard_index.search(xq, k)
+
+        # Nearest first within each result row...
+        assert np.all(D[:, :-1] <= D[:, 1:]), "sharded METRIC_L1 results are not sorted by distance"
+        # ...and the same neighbors the unsharded index finds. Distances are compared rather than
+        # labels so that equidistant neighbors may be returned in either order.
+        np.testing.assert_array_almost_equal(D, Dref, decimal=5)
+
     def test_shards_ivf(self):
         ds = SyntheticDataset(32, 1000, 100, 20)
         ref_index = faiss.index_factory(ds.d, "IVF32,SQ8")