|
| 1 | +# Copyright (c) Meta Platforms, Inc. and affiliates. |
| 2 | +# |
| 3 | +# This source code is licensed under the MIT license found in the |
| 4 | +# LICENSE file in the root directory of this source tree. |
| 5 | + |
| 6 | +""" |
| 7 | +Benchmark script for testing HNSW prune_headroom recall impact. |
| 8 | +
|
| 9 | +Compares recall and build time between different prune_headroom values. |
| 10 | +Default comparison is between: |
| 11 | +- Baseline: prune_headroom = 0.0 (original behavior, no headroom) |
| 12 | +- With headroom: prune_headroom = 0.2 (proposed default) |
| 13 | +
|
| 14 | +Usage: |
| 15 | + python bench_hnsw_prune_headroom.py |
| 16 | + python bench_hnsw_prune_headroom.py --nb 100000 --d 256 |
| 17 | + python bench_hnsw_prune_headroom.py --headroom_values 0.0 0.1 0.2 0.3 |
| 18 | +""" |
| 19 | + |
| 20 | +import argparse |
| 21 | +import time |
| 22 | + |
| 23 | +import faiss |
| 24 | + |
| 25 | +try: |
| 26 | + from faiss.contrib.datasets_fb import DatasetSIFT1M |
| 27 | +except ImportError: |
| 28 | + from faiss.contrib.datasets import DatasetSIFT1M |
| 29 | + |
| 30 | +from faiss.contrib.datasets import SyntheticDataset |
| 31 | + |
| 32 | + |
| 33 | +def compute_recall(I, gt, k): |
| 34 | + """Compute recall@k given search results I and ground truth gt.""" |
| 35 | + nq = gt.shape[0] |
| 36 | + return faiss.eval_intersection(I[:, :k], gt[:, :k]) / (nq * k) |
| 37 | + |
| 38 | + |
| 39 | +def build_hnsw_index(d, m, xb, ef_construction, prune_headroom): |
| 40 | + """Build an HNSW index with the specified configuration.""" |
| 41 | + index = faiss.IndexHNSWSQ(d, faiss.ScalarQuantizer.QT_4bit, m) |
| 42 | + index.hnsw.efConstruction = ef_construction |
| 43 | + index.hnsw.prune_headroom = prune_headroom |
| 44 | + |
| 45 | + index.train(xb) |
| 46 | + start_time = time.time() |
| 47 | + index.add(xb) |
| 48 | + build_time = time.time() - start_time |
| 49 | + |
| 50 | + return index, build_time |
| 51 | + |
| 52 | + |
| 53 | +def run_benchmark( |
| 54 | + d=384, |
| 55 | + m=32, |
| 56 | + nb=50000, |
| 57 | + nq=1000, |
| 58 | + reps=3, |
| 59 | + ef_construction=40, |
| 60 | + ef_search_values=None, |
| 61 | + k_values=None, |
| 62 | + headroom_values=None, |
| 63 | + use_sift1m=False, |
| 64 | +): |
| 65 | + """ |
| 66 | + Run the prune_headroom recall benchmark. |
| 67 | +
|
| 68 | + Args: |
| 69 | + d: Dimension of vectors |
| 70 | + nb: Number of base vectors |
| 71 | + nq: Number of query vectors |
| 72 | + ef_construction: efConstruction parameter for HNSW |
| 73 | + ef_search_values: List of efSearch values to test |
| 74 | + k_values: List of k values for recall@k |
| 75 | + headroom_values: List of prune_headroom values to compare |
| 76 | + use_sift1m: Use SIFT1M dataset instead of synthetic |
| 77 | +
|
| 78 | + Returns: |
| 79 | + Dictionary containing benchmark results |
| 80 | + """ |
| 81 | + if ef_search_values is None: |
| 82 | + ef_search_values = [16, 32, 64, 128, 256] |
| 83 | + if k_values is None: |
| 84 | + k_values = [1, 10] |
| 85 | + if headroom_values is None: |
| 86 | + headroom_values = [0.0, 0.2] |
| 87 | + |
| 88 | + if use_sift1m: |
| 89 | + print("Loading SIFT1M dataset") |
| 90 | + ds = DatasetSIFT1M() |
| 91 | + xb = ds.get_database() |
| 92 | + xq = ds.get_queries() |
| 93 | + d = xb.shape[1] |
| 94 | + nb = xb.shape[0] |
| 95 | + nq = xq.shape[0] |
| 96 | + else: |
| 97 | + print(f"Generating synthetic dataset: d={d}, nb={nb}, nq={nq}") |
| 98 | + ds = SyntheticDataset(d=d, nt=0, nb=nb, nq=nq) |
| 99 | + xb = ds.get_database() |
| 100 | + xq = ds.get_queries() |
| 101 | + |
| 102 | + max_k = max(k_values) |
| 103 | + print(f"Computing ground truth for k={max_k}") |
| 104 | + gt = ds.get_groundtruth(k=max_k) |
| 105 | + |
| 106 | + results = {"build_times": {}, "ndis_search": {}, "recalls": {}} |
| 107 | + |
| 108 | + for headroom in headroom_values: |
| 109 | + for rep in range(reps): |
| 110 | + index, build_time = build_hnsw_index( |
| 111 | + d, m, xb, ef_construction, headroom) |
| 112 | + results["build_times"][headroom] = build_time |
| 113 | + |
| 114 | + faiss.cvar.hnsw_stats.reset() |
| 115 | + row = {} |
| 116 | + results["recalls"][(headroom, rep)] = row |
| 117 | + for ef_search in ef_search_values: |
| 118 | + index.hnsw.efSearch = ef_search |
| 119 | + _, I = index.search(xq, max_k) |
| 120 | + |
| 121 | + col = {} |
| 122 | + row[ef_search] = col |
| 123 | + for k in k_values: |
| 124 | + recall = compute_recall(I, gt, k) |
| 125 | + col[k] = recall |
| 126 | + ndis_search = faiss.cvar.hnsw_stats.ndis / nq |
| 127 | + results["ndis_search"][headroom] = ndis_search |
| 128 | + print( |
| 129 | + f"HNSW{m}(prune_headroom={headroom:4.2f}): " |
| 130 | + f"{build_time=:4.2f}s, {ndis_search=:5.1f}" |
| 131 | + ) |
| 132 | + |
| 133 | + print_results_table(results, ef_search_values, k_values, headroom_values) |
| 134 | + return results |
| 135 | + |
| 136 | + |
| 137 | +def print_results_table(results, ef_search_values, k_values, headroom_values): |
| 138 | + |
| 139 | + for k in k_values: |
| 140 | + header_parts = [f"{k=:2} "] |
| 141 | + for ef_search in ef_search_values: |
| 142 | + header_parts.append(f"ef={ef_search:3}") |
| 143 | + header = " | ".join(header_parts) |
| 144 | + |
| 145 | + print(f"\n{header}") |
| 146 | + print("-" * len(header)) |
| 147 | + for (h, _), row in results["recalls"].items(): |
| 148 | + row_parts = [f"h={h:4.2f}"] |
| 149 | + for ef_search in ef_search_values: |
| 150 | + recall = row[ef_search][k] |
| 151 | + row_parts.append(f"{recall:6.4f}") |
| 152 | + print(" | ".join(row_parts)) |
| 153 | + |
| 154 | + |
| 155 | +if __name__ == "__main__": |
| 156 | + parser = argparse.ArgumentParser( |
| 157 | + description="HNSW prune_headroom recall and build time benchmark" |
| 158 | + ) |
| 159 | + parser.add_argument( |
| 160 | + "--d", |
| 161 | + type=int, |
| 162 | + default=128, |
| 163 | + help="Dimension of vectors (default: 128)", |
| 164 | + ) |
| 165 | + parser.add_argument( |
| 166 | + "--m", |
| 167 | + type=int, |
| 168 | + default=32, |
| 169 | + help="Node degree (M, default: 32)", |
| 170 | + ) |
| 171 | + parser.add_argument( |
| 172 | + "--nb", |
| 173 | + type=int, |
| 174 | + default=50000, |
| 175 | + help="Number of base vectors (default: 50000)", |
| 176 | + ) |
| 177 | + parser.add_argument( |
| 178 | + "--nq", |
| 179 | + type=int, |
| 180 | + default=10000, |
| 181 | + help="Number of query vectors (default: 10000)", |
| 182 | + ) |
| 183 | + parser.add_argument( |
| 184 | + "--ef_construction", |
| 185 | + type=int, |
| 186 | + default=40, |
| 187 | + help="efConstruction parameter (default: 40)", |
| 188 | + ) |
| 189 | + parser.add_argument( |
| 190 | + "--ef_search", |
| 191 | + type=int, |
| 192 | + nargs="+", |
| 193 | + default=[16, 32, 64, 128, 256], |
| 194 | + help="efSearch values to test (default: 16 32 64 128 256)", |
| 195 | + ) |
| 196 | + parser.add_argument( |
| 197 | + "--k", |
| 198 | + type=int, |
| 199 | + nargs="+", |
| 200 | + default=[1, 10], |
| 201 | + help="k values for recall@k (default: 1 10)", |
| 202 | + ) |
| 203 | + parser.add_argument( |
| 204 | + "--headroom_values", |
| 205 | + type=float, |
| 206 | + nargs="+", |
| 207 | + default=[0.0, 0.04, 0.08, 0.12, 0.16, 0.20], |
| 208 | + help="prune_headroom values to compare (default: 0.0 0.2)", |
| 209 | + ) |
| 210 | + parser.add_argument( |
| 211 | + "--reps", |
| 212 | + type=int, |
| 213 | + default=3, |
| 214 | + help="Number of repetitions (default: 3)", |
| 215 | + ) |
| 216 | + parser.add_argument( |
| 217 | + "--sift1m", |
| 218 | + action="store_true", |
| 219 | + help="Use SIFT1M dataset instead of synthetic", |
| 220 | + ) |
| 221 | + args = parser.parse_args() |
| 222 | + |
| 223 | + run_benchmark( |
| 224 | + d=args.d, |
| 225 | + m=args.m, |
| 226 | + nb=args.nb, |
| 227 | + nq=args.nq, |
| 228 | + reps=args.reps, |
| 229 | + ef_construction=args.ef_construction, |
| 230 | + ef_search_values=args.ef_search, |
| 231 | + k_values=args.k, |
| 232 | + headroom_values=args.headroom_values, |
| 233 | + use_sift1m=args.sift1m, |
| 234 | + ) |
0 commit comments