Skip to content

Commit af23585

Browse files
divyegaladivyegala
committed
Merge remote-tracking branch 'origin/main' into jit-lto-ivf-flat-interleaved
2 parents 533b770 + c548f22 commit af23585

13 files changed

Lines changed: 31 additions & 35 deletions

File tree

‎.github/workflows/build.yaml‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,6 @@ jobs:
4343
node_type: cpu16
4444
script: ci/build_cpp.sh
4545
sha: ${{ inputs.sha }}
46-
sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
4746

4847
rocky8-clib-standalone-build:
4948
secrets: inherit
@@ -69,7 +68,6 @@ jobs:
6968
artifact-name: "libcuvs_c_${{ matrix.cuda_version }}_${{ matrix.arch }}.tar.gz"
7069
file_to_upload: "libcuvs_c.tar.gz"
7170
sha: ${{ inputs.sha }}
72-
sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
7371
rust-build:
7472
needs: cpp-build
7573
secrets: inherit
@@ -91,7 +89,6 @@ jobs:
9189
node_type: "gpu-l4-latest-1"
9290
script: "ci/build_rust.sh"
9391
sha: ${{ inputs.sha }}
94-
sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
9592
go-build:
9693
needs: cpp-build
9794
secrets: inherit
@@ -135,7 +132,6 @@ jobs:
135132
artifact-name: "cuvs-java-cuda${{ matrix.cuda_version }}"
136133
file_to_upload: "java/cuvs-java/target/"
137134
sha: ${{ inputs.sha }}
138-
sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
139135
python-build:
140136
needs: [cpp-build]
141137
secrets: inherit
@@ -146,7 +142,6 @@ jobs:
146142
date: ${{ inputs.date }}
147143
script: ci/build_python.sh
148144
sha: ${{ inputs.sha }}
149-
sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
150145
upload-conda:
151146
needs: [cpp-build, python-build]
152147
secrets: inherit
@@ -189,7 +184,6 @@ jobs:
189184
matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber)))
190185
package-name: libcuvs
191186
package-type: cpp
192-
sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
193187
wheel-publish-libcuvs:
194188
needs: wheel-build-libcuvs
195189
secrets: inherit
@@ -213,7 +207,6 @@ jobs:
213207
node_type: cpu8
214208
script: ci/build_wheel_cuvs.sh
215209
package-name: cuvs
216-
sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
217210
package-type: python
218211
wheel-publish-cuvs:
219212
needs: wheel-build-cuvs

‎.github/workflows/pr.yaml‎

Lines changed: 0 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -317,7 +317,6 @@ jobs:
317317
build_type: pull-request
318318
node_type: cpu16
319319
script: ci/build_cpp.sh
320-
sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
321320
conda-cpp-tests:
322321
needs: [conda-cpp-build, changed-files]
323322
secrets: inherit
@@ -326,7 +325,6 @@ jobs:
326325
with:
327326
build_type: pull-request
328327
script: ci/test_cpp.sh
329-
sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
330328
conda-cpp-checks:
331329
needs: conda-cpp-build
332330
secrets: inherit
@@ -349,7 +347,6 @@ jobs:
349347
with:
350348
build_type: pull-request
351349
script: ci/test_python.sh
352-
sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
353350
rocky8-clib-standalone-build:
354351
needs: [checks]
355352
secrets: inherit
@@ -374,7 +371,6 @@ jobs:
374371
artifact-name: "libcuvs_c_${{ matrix.cuda_version }}_${{ matrix.arch }}.tar.gz"
375372
file_to_upload: "libcuvs_c.tar.gz"
376373
sha: ${{ inputs.sha }}
377-
sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
378374
rocky8-clib-tests:
379375
needs: [rocky8-clib-standalone-build, changed-files]
380376
secrets: inherit
@@ -418,7 +414,6 @@ jobs:
418414
script: "ci/test_java.sh"
419415
artifact-name: "cuvs-java-cuda${{ matrix.cuda_version }}"
420416
file_to_upload: "java/cuvs-java/target/"
421-
sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
422417
rust-build:
423418
needs: [conda-cpp-build, changed-files]
424419
secrets: inherit
@@ -438,7 +433,6 @@ jobs:
438433
arch: "amd64"
439434
container_image: "rapidsai/ci-conda:26.04-cuda${{ matrix.cuda_version }}-ubuntu24.04-py3.13"
440435
script: "ci/build_rust.sh"
441-
sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
442436
go-build:
443437
needs: [conda-cpp-build, changed-files]
444438
secrets: inherit
@@ -480,7 +474,6 @@ jobs:
480474
matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber)))
481475
package-name: libcuvs
482476
package-type: cpp
483-
sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
484477
wheel-build-cuvs:
485478
needs: wheel-build-libcuvs
486479
secrets: inherit
@@ -491,7 +484,6 @@ jobs:
491484
script: ci/build_wheel_cuvs.sh
492485
package-name: cuvs
493486
package-type: python
494-
sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
495487
wheel-tests-cuvs:
496488
needs: [wheel-build-cuvs, changed-files]
497489
secrets: inherit
@@ -500,7 +492,6 @@ jobs:
500492
with:
501493
build_type: pull-request
502494
script: ci/test_wheel_cuvs.sh
503-
sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
504495
devcontainer:
505496
secrets: inherit
506497
needs: telemetry-setup
@@ -509,12 +500,10 @@ jobs:
509500
arch: '["amd64", "arm64"]'
510501
cuda: '["13.1"]'
511502
node_type: "cpu8"
512-
rapids-aux-secret-1: GIST_REPO_READ_ORG_GITHUB_TOKEN
513503
env: |
514504
SCCACHE_DIST_MAX_RETRIES=inf
515505
SCCACHE_SERVER_LOG=sccache=debug
516506
SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE=false
517-
SCCACHE_DIST_AUTH_TOKEN_VAR=RAPIDS_AUX_SECRET_1
518507
build_command: |
519508
sccache --zero-stats;
520509
build-all -j0 --verbose 2>&1 | tee telemetry-artifacts/build.log;

‎.github/workflows/test.yaml‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,6 @@ jobs:
4141
date: ${{ inputs.date }}
4242
script: ci/test_cpp.sh
4343
sha: ${{ inputs.sha }}
44-
sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
4544
conda-python-tests:
4645
secrets: inherit
4746
uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@main
@@ -51,7 +50,6 @@ jobs:
5150
date: ${{ inputs.date }}
5251
script: ci/test_python.sh
5352
sha: ${{ inputs.sha }}
54-
sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
5553
conda-java-tests:
5654
secrets: inherit
5755
uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main
@@ -72,7 +70,6 @@ jobs:
7270
arch: "amd64"
7371
container_image: "rapidsai/ci-conda:26.04-cuda${{ matrix.cuda_version }}-ubuntu24.04-py3.13"
7472
script: "ci/test_java.sh"
75-
sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
7673
wheel-tests-cuvs:
7774
secrets: inherit
7875
uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@main
@@ -82,4 +79,3 @@ jobs:
8279
date: ${{ inputs.date }}
8380
sha: ${{ inputs.sha }}
8481
script: ci/test_wheel_cuvs.sh
85-
sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN

‎c/src/neighbors/hnsw.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -169,7 +169,7 @@ extern "C" cuvsError_t cuvsHnswAceParamsDestroy(cuvsHnswAceParams_t params)
169169
extern "C" cuvsError_t cuvsHnswIndexParamsCreate(cuvsHnswIndexParams_t* params)
170170
{
171171
return cuvs::core::translate_exceptions([=] {
172-
*params = new cuvsHnswIndexParams{.hierarchy = cuvsHnswHierarchy::NONE,
172+
*params = new cuvsHnswIndexParams{.hierarchy = cuvsHnswHierarchy::GPU,
173173
.ef_construction = 200,
174174
.num_threads = 0,
175175
.M = 32,

‎c/tests/neighbors/ann_hnsw_c.cu‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,8 @@ TEST(CagraHnswC, BuildSearch)
102102
hnsw_index->dtype = index->dtype;
103103
cuvsHnswIndexParams_t hnsw_params;
104104
cuvsHnswIndexParamsCreate(&hnsw_params);
105+
// Use NONE hierarchy since cuvsCagraSerializeToHnswlib creates a base-layer-only index
106+
hnsw_params->hierarchy = NONE;
105107
cuvsHnswDeserialize(res, hnsw_params, "/tmp/cagra_hnswlib.index", 2, L2Expanded, hnsw_index);
106108

107109
// search index

‎cpp/include/cuvs/neighbors/hnsw.hpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ enum class HnswHierarchy {
4545

4646
struct index_params : cuvs::neighbors::index_params {
4747
/** Hierarchy build type for HNSW index when converting from CAGRA index */
48-
HnswHierarchy hierarchy = HnswHierarchy::NONE;
48+
HnswHierarchy hierarchy = HnswHierarchy::GPU;
4949
/** Size of the candidate list during hierarchy construction when hierarchy is `CPU`*/
5050
int ef_construction = 200;
5151
/** Number of host threads to use to construct hierarchy when hierarchy is `CPU` or `GPU`.

‎examples/cpp/src/cagra_hnsw_ace_example.cu‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,8 @@ void cagra_build_search_ace(raft::device_resources const& dev_resources,
8080
// descriptor For in-memory indices: creates HNSW index in memory
8181
std::cout << "Converting CAGRA index to HNSW" << std::endl;
8282
hnsw::index_params hnsw_params;
83-
auto hnsw_index = hnsw::from_cagra(dev_resources, hnsw_params, index);
83+
hnsw_params.hierarchy = hnsw::HnswHierarchy::GPU; // Offload hierarchy construction to GPU
84+
auto hnsw_index = hnsw::from_cagra(dev_resources, hnsw_params, index);
8485

8586
// HNSW search requires host matrices
8687
auto queries_host = raft::make_host_matrix<float, int64_t>(n_queries, queries.extent(1));

‎java/cuvs-java/src/main/java/com/nvidia/cuvs/HnswIndexParams.java‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,16 @@ private CuvsHnswHierarchy(int value) {
5858
}
5959
};
6060

61-
private CuvsHnswHierarchy hierarchy = CuvsHnswHierarchy.NONE;
61+
/**
62+
* Alias for {@link CuvsHnswHierarchy} for convenience.
63+
*/
64+
public static class HnswHierarchy {
65+
public static final CuvsHnswHierarchy NONE = CuvsHnswHierarchy.NONE;
66+
public static final CuvsHnswHierarchy CPU = CuvsHnswHierarchy.CPU;
67+
public static final CuvsHnswHierarchy GPU = CuvsHnswHierarchy.GPU;
68+
}
69+
70+
private CuvsHnswHierarchy hierarchy = CuvsHnswHierarchy.GPU;
6271
private int efConstruction = 200;
6372
private int numThreads = 2;
6473
private int vectorDimension;
@@ -168,7 +177,7 @@ public String toString() {
168177
*/
169178
public static class Builder {
170179

171-
private CuvsHnswHierarchy hierarchy = CuvsHnswHierarchy.NONE;
180+
private CuvsHnswHierarchy hierarchy = CuvsHnswHierarchy.GPU;
172181
private int efConstruction = 200;
173182
private int numThreads = 2;
174183
private int vectorDimension;

‎java/cuvs-java/src/main/java22/com/nvidia/cuvs/internal/HnswIndexImpl.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -222,6 +222,7 @@ private IndexReference deserialize(InputStream inputStream) throws Throwable {
222222
*/
223223
private CloseableHandle segmentFromIndexParams(HnswIndexParams params) {
224224
var hnswParams = createHnswIndexParams();
225+
cuvsHnswIndexParams.hierarchy(hnswParams.handle(), params.getHierarchy().value);
225226
cuvsHnswIndexParams.ef_construction(hnswParams.handle(), params.getEfConstruction());
226227
cuvsHnswIndexParams.num_threads(hnswParams.handle(), params.getNumThreads());
227228
return hnswParams;

‎java/cuvs-java/src/test/java/com/nvidia/cuvs/CagraAceBuildAndSearchIT.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
import com.carrotsearch.randomizedtesting.RandomizedRunner;
1111
import com.nvidia.cuvs.CagraIndexParams.CagraGraphBuildAlgo;
1212
import com.nvidia.cuvs.CagraIndexParams.CuvsDistanceType;
13+
import com.nvidia.cuvs.HnswIndexParams.HnswHierarchy;
1314
import java.nio.file.Files;
1415
import java.nio.file.Path;
1516
import java.util.Arrays;
@@ -186,7 +187,7 @@ public void testAceDiskBasedBuild() throws Throwable {
186187
// Convert CAGRA index to HNSW using fromCagra
187188
// This automatically handles disk-based indices
188189
HnswIndexParams hnswIndexParams =
189-
new HnswIndexParams.Builder().withVectorDimension(2).build();
190+
new HnswIndexParams.Builder().withVectorDimension(2).withHierarchy(HnswHierarchy.GPU).build();
190191

191192
try (var hnswIndexSerialized = HnswIndex.fromCagra(hnswIndexParams, index)) {
192193
var hnswIndexSerializedPath = buildDir.resolve("hnsw_index.bin");

0 commit comments

Comments
 (0)