Skip to content

Commit 75134fa

Browse files
SpadeA-TangSpadeA-Tang
committed
rebase
Signed-off-by: SpadeA <tangchenjie1210@gmail.com>
2 parents 1870f57 + 6eebfe4 commit 75134fa

181 files changed

Lines changed: 15686 additions & 2018 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/actions/cache-restore/action.yaml‎

Lines changed: 37 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,40 @@ inputs:
99
runs:
1010
using: 'composite'
1111
steps:
12+
- name: Detect CPU Feature Bucket
13+
id: cpu_feature
14+
shell: bash
15+
run: |
16+
set -euo pipefail
17+
18+
flags=""
19+
if command -v lscpu >/dev/null 2>&1; then
20+
flags="$(LC_ALL=C lscpu | awk -F: '/^(Flags|Features):/ {gsub(/^[[:space:]]+/, "", $2); print $2; exit}' | tr '[:upper:]' '[:lower:]')"
21+
fi
22+
23+
if [ -z "${flags}" ] && command -v sysctl >/dev/null 2>&1; then
24+
flags="$(sysctl -a machdep.cpu.features machdep.cpu.leaf7_features 2>/dev/null | tr '[:upper:]' '[:lower:]' || true)"
25+
fi
26+
27+
if [ -z "${flags}" ] && [ -r /proc/cpuinfo ]; then
28+
flags="$(awk -F: '/^(flags|Features)[[:space:]]*:/ {gsub(/^[[:space:]]+/, "", $2); print $2; exit}' /proc/cpuinfo | tr '[:upper:]' '[:lower:]')"
29+
fi
30+
31+
features=()
32+
for feature in avx512f avx512bw avx512vl avx512dq avx512cd avx512vnni avx512vbmi2 avx512vpopcntdq avx512_bf16 avx512_fp16 avx2 fma sse4_2 sve asimd; do
33+
if grep -qw "${feature}" <<< "${flags}"; then
34+
features+=("${feature}")
35+
fi
36+
done
37+
38+
if [ ${#features[@]} -eq 0 ]; then
39+
bucket="cpu-generic"
40+
else
41+
bucket="cpu-$(IFS=_; echo "${features[*]}")"
42+
fi
43+
44+
echo "bucket=${bucket}" >> "${GITHUB_OUTPUT}"
45+
echo "Using cache CPU feature bucket: ${bucket}"
1246
- name: Cache CCache Volumes
1347
uses: actions/cache/restore@v4
1448
with:
@@ -19,13 +53,13 @@ runs:
1953
# inside the restored tarball still detects thirdparty source changes
2054
# via its own content hashing, so compilation correctness is unaffected.
2155
# Key must match cache-save/action.yaml exactly for exact-key hits.
22-
key: knowhere-${{ inputs.os }}-${{ github.job }}-ccache-${{ hashFiles('src/**/*.cpp', 'src/**/*.cc', 'src/**/*.c', 'src/**/*.h', 'src/**/*.hpp', 'include/**/*.h', 'include/**/*.hpp', 'tests/**/*.cpp', 'tests/**/*.cc', 'tests/**/*.h', 'benchmark/**/*.cpp', 'benchmark/**/*.cc', 'benchmark/**/*.h', 'cmake/**', 'CMakeLists.txt', 'Makefile', 'conanfile.py') }}
23-
restore-keys: knowhere-${{ inputs.os }}-${{ github.job }}-ccache-
56+
key: knowhere-${{ inputs.os }}-${{ github.job }}-${{ steps.cpu_feature.outputs.bucket }}-ccache-${{ hashFiles('src/**/*.cpp', 'src/**/*.cc', 'src/**/*.c', 'src/**/*.h', 'src/**/*.hpp', 'include/**/*.h', 'include/**/*.hpp', 'tests/**/*.cpp', 'tests/**/*.cc', 'tests/**/*.h', 'benchmark/**/*.cpp', 'benchmark/**/*.cc', 'benchmark/**/*.h', 'cmake/**', 'CMakeLists.txt', 'Makefile', 'conanfile.py') }}
57+
restore-keys: knowhere-${{ inputs.os }}-${{ github.job }}-${{ steps.cpu_feature.outputs.bucket }}-ccache-
2458
- name: Reset Conan package cache
2559
shell: bash
2660
run: rm -rf ~/.conan2/p
2761
- name: Cache Conan Packages
2862
uses: actions/cache/restore@v4
2963
with:
3064
path: ~/.conan2
31-
key: knowhere-${{ inputs.os }}-${{ github.job }}-conan2-${{ hashFiles('conanfile.py') }}
65+
key: knowhere-${{ inputs.os }}-${{ github.job }}-${{ steps.cpu_feature.outputs.bucket }}-conan2-${{ hashFiles('conanfile.py') }}

‎.github/actions/cache-save/action.yaml‎

Lines changed: 36 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,40 @@ inputs:
99
runs:
1010
using: 'composite'
1111
steps:
12+
- name: Detect CPU Feature Bucket
13+
id: cpu_feature
14+
shell: bash
15+
run: |
16+
set -euo pipefail
17+
18+
flags=""
19+
if command -v lscpu >/dev/null 2>&1; then
20+
flags="$(LC_ALL=C lscpu | awk -F: '/^(Flags|Features):/ {gsub(/^[[:space:]]+/, "", $2); print $2; exit}' | tr '[:upper:]' '[:lower:]')"
21+
fi
22+
23+
if [ -z "${flags}" ] && command -v sysctl >/dev/null 2>&1; then
24+
flags="$(sysctl -a machdep.cpu.features machdep.cpu.leaf7_features 2>/dev/null | tr '[:upper:]' '[:lower:]' || true)"
25+
fi
26+
27+
if [ -z "${flags}" ] && [ -r /proc/cpuinfo ]; then
28+
flags="$(awk -F: '/^(flags|Features)[[:space:]]*:/ {gsub(/^[[:space:]]+/, "", $2); print $2; exit}' /proc/cpuinfo | tr '[:upper:]' '[:lower:]')"
29+
fi
30+
31+
features=()
32+
for feature in avx512f avx512bw avx512vl avx512dq avx512cd avx512vnni avx512vbmi2 avx512vpopcntdq avx512_bf16 avx512_fp16 avx2 fma sse4_2 sve asimd; do
33+
if grep -qw "${feature}" <<< "${flags}"; then
34+
features+=("${feature}")
35+
fi
36+
done
37+
38+
if [ ${#features[@]} -eq 0 ]; then
39+
bucket="cpu-generic"
40+
else
41+
bucket="cpu-$(IFS=_; echo "${features[*]}")"
42+
fi
43+
44+
echo "bucket=${bucket}" >> "${GITHUB_OUTPUT}"
45+
echo "Using cache CPU feature bucket: ${bucket}"
1246
- name: Cache CCache Volumes
1347
uses: actions/cache/save@v4
1448
with:
@@ -19,9 +53,9 @@ runs:
1953
# inside the restored tarball still detects thirdparty source changes
2054
# via its own content hashing, so compilation correctness is unaffected.
2155
# Key must match cache-restore/action.yaml exactly for exact-key hits.
22-
key: knowhere-${{ inputs.os }}-${{ github.job }}-ccache-${{ hashFiles('src/**/*.cpp', 'src/**/*.cc', 'src/**/*.c', 'src/**/*.h', 'src/**/*.hpp', 'include/**/*.h', 'include/**/*.hpp', 'tests/**/*.cpp', 'tests/**/*.cc', 'tests/**/*.h', 'benchmark/**/*.cpp', 'benchmark/**/*.cc', 'benchmark/**/*.h', 'cmake/**', 'CMakeLists.txt', 'Makefile', 'conanfile.py') }}
56+
key: knowhere-${{ inputs.os }}-${{ github.job }}-${{ steps.cpu_feature.outputs.bucket }}-ccache-${{ hashFiles('src/**/*.cpp', 'src/**/*.cc', 'src/**/*.c', 'src/**/*.h', 'src/**/*.hpp', 'include/**/*.h', 'include/**/*.hpp', 'tests/**/*.cpp', 'tests/**/*.cc', 'tests/**/*.h', 'benchmark/**/*.cpp', 'benchmark/**/*.cc', 'benchmark/**/*.h', 'cmake/**', 'CMakeLists.txt', 'Makefile', 'conanfile.py') }}
2357
- name: Cache Conan Packages
2458
uses: actions/cache/save@v4
2559
with:
2660
path: ~/.conan2
27-
key: knowhere-${{ inputs.os }}-${{ github.job }}-conan2-${{ hashFiles('conanfile.py') }}
61+
key: knowhere-${{ inputs.os }}-${{ github.job }}-${{ steps.cpu_feature.outputs.bucket }}-conan2-${{ hashFiles('conanfile.py') }}

‎.github/mergify.yml‎

Lines changed: 19 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ pull_request_rules:
33
conditions:
44
- or:
55
- base=main
6-
- base~=^2(\.\d+){1,2}$
6+
- base~=^\d+(\.\d+){1,2}$
77
- -status-success=DCO
88
actions:
99
label:
@@ -16,7 +16,7 @@ pull_request_rules:
1616
conditions:
1717
- or:
1818
- base=main
19-
- base~=^2(\.\d+){1,2}$
19+
- base~=^\d+(\.\d+){1,2}$
2020
- status-success=DCO
2121
actions:
2222
label:
@@ -25,12 +25,11 @@ pull_request_rules:
2525
add:
2626
- dco-passed
2727

28-
2928
- name: Blocking PR if missing a related issue or PR doesn't have kind/improvement label
3029
conditions:
3130
- or:
3231
- base=main
33-
- base~=^2(\.\d+){1,2}$
32+
- base~=^\d+(\.\d+){1,2}$
3433
- title!=\[automated\]
3534
- label!=kind/improvement
3635
- and:
@@ -55,12 +54,11 @@ pull_request_rules:
5554
5655
Thanks for your efforts and contribution to the community!.
5756
58-
5957
- name: Dismiss block label if related issue be added into PR
6058
conditions:
6159
- or:
6260
- base=main
63-
- base~=^2(\.\d+){1,2}$
61+
- base~=^\d+(\.\d+){1,2}$
6462
- label!=kind/improvement
6563
- and:
6664
- or:
@@ -79,7 +77,7 @@ pull_request_rules:
7977
conditions:
8078
- or:
8179
- base=main
82-
- base~=^2(\.\d+){1,2}$
80+
- base~=^\d+(\.\d+){1,2}$
8381
- or:
8482
- and:
8583
- label=kind/improvement
@@ -113,12 +111,11 @@ pull_request_rules:
113111
message: |
114112
@{{author}} Please remove redundant `kind/xxx` labels and make sure that there is only one `kind/xxx` label of your Pull Request. (eg. “/remove-kind improvement”)
115113
116-
117114
- name: Dismiss block PR if have not multiple 'kind' type labels
118115
conditions:
119116
- or:
120117
- base=main
121-
- base~=^2(\.\d+){1,2}$
118+
- base~=^\d+(\.\d+){1,2}$
122119
- or:
123120
- and:
124121
- label=kind/improvement
@@ -150,12 +147,11 @@ pull_request_rules:
150147
remove:
151148
- do-not-merge/description-tag-conflict
152149

153-
154-
- name: Test passed for code changed for main
150+
- name: Test passed for code changed
155151
conditions:
156152
- or:
157153
- base=main
158-
- base~=^2(\.\d+){1,2}$
154+
- base~=^\d+(\.\d+){1,2}$
159155
- "status-success=ut on ubuntu-22.04"
160156
- "status-success=ut on macos-15"
161157
- 'status-success=pre-commit'
@@ -170,11 +166,11 @@ pull_request_rules:
170166
add:
171167
- ci-passed
172168

173-
- name: Test passed for cpu code changed only for main
169+
- name: Test passed for cpu code changed only
174170
conditions:
175171
- or:
176172
- base=main
177-
- base~=^2(\.\d+){1,2}$
173+
- base~=^\d+(\.\d+){1,2}$
178174
- -files~=^(?=src/index/ivf_raft/)
179175
- files~=^(?=.*((\.(h|cpp)|CMakeLists.txt))).*$
180176
- "status-success=ut on ubuntu-22.04"
@@ -189,11 +185,11 @@ pull_request_rules:
189185
add:
190186
- ci-passed
191187

192-
- name: Test passed for non c++ or go code changed for main
188+
- name: Test passed for non c++ or go code changed
193189
conditions:
194190
- or:
195191
- base=main
196-
- base~=^2(\.\d+){1,2}$
192+
- base~=^\d+(\.\d+){1,2}$
197193
- -files~=^(?=.*((\.(h|cpp)|CMakeLists.txt))).*$
198194
- 'status-success=pre-commit'
199195
- 'status-success=analyzer'
@@ -208,7 +204,7 @@ pull_request_rules:
208204
conditions:
209205
- or:
210206
- base=main
211-
- base~=^2(\.\d+){1,2}$
207+
- base~=^\d+(\.\d+){1,2}$
212208
- title~=\[skip e2e\]
213209
- -files~=^(?=.*((\.(h|cpp)|CMakeLists.txt))).*$
214210
actions:
@@ -220,7 +216,7 @@ pull_request_rules:
220216
conditions:
221217
- or:
222218
- base=main
223-
- base~=^2(\.\d+){1,2}$
219+
- base~=^\d+(\.\d+){1,2}$
224220
- 'check-failure=e2e'
225221
actions:
226222
comment:
@@ -231,18 +227,18 @@ pull_request_rules:
231227
conditions:
232228
- or:
233229
- base=main
234-
- base~=^2(\.\d+){1,2}$
230+
- base~=^\d+(\.\d+){1,2}$
235231
- 'check-failure=e2e-sse'
236232
actions:
237233
comment:
238234
message: |
239235
@{{author}} e2e jenkins job failed, comment `/run-e2e-sse` can trigger the job again.
240236
241-
- name: Remove ci-passed label when ut failed for main
237+
- name: Remove ci-passed label when ut failed
242238
conditions:
243239
- or:
244240
- base=main
245-
- base~=^2(\.\d+){1,2}$
241+
- base~=^\d+(\.\d+){1,2}$
246242
- or:
247243
- "status-success!=ut on ubuntu-22.04"
248244
- "status-success!=ut on macos-15"
@@ -257,11 +253,11 @@ pull_request_rules:
257253
remove:
258254
- ci-passed
259255

260-
- name: Remove ci-passed label when gpu ut failed for main
256+
- name: Remove ci-passed label when gpu ut failed
261257
conditions:
262258
- or:
263259
- base=main
264-
- base~=^2(\.\d+){1,2}$
260+
- base~=^\d+(\.\d+){1,2}$
265261
- or:
266262
- "status-success!=ut on ubuntu-22.04"
267263
- "status-success!=ut on macos-15"

‎CMakeLists.txt‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -191,8 +191,6 @@ if(WITH_CARDINAL)
191191
list(APPEND CARDINAL_LIBS cardinalv1)
192192
add_subdirectory(cmake/libs/cardinal/v2)
193193
list(APPEND CARDINAL_LIBS cardinalv2)
194-
knowhere_file_glob(GLOB_RECURSE KNOWHERE_SPARSE_NODE_SRCS src/index/sparse/sparse_index_node.cc)
195-
list(REMOVE_ITEM KNOWHERE_SRCS ${KNOWHERE_SPARSE_NODE_SRCS})
196194
endif()
197195

198196
# Sparse codec C files (not captured by *.cc globs)
@@ -288,6 +286,9 @@ target_link_libraries(knowhere PUBLIC ${KNOWHERE_LINKER_LIBS})
288286
target_link_libraries(knowhere PUBLIC Boost::boost nlohmann_json::nlohmann_json glog::glog Folly::folly prometheus-cpp::core opentelemetry-cpp::opentelemetry_trace)
289287

290288
if (WITH_CARDINAL)
289+
target_compile_definitions(knowhere PRIVATE
290+
SPARSE_INDEX_VERSION_USE_RAW_DATA_THRESHOLD=7
291+
SPARSE_INDEX_VERSION_SUPPORT_FP16_QUANT_FOR_IP=0)
291292
target_link_libraries(knowhere PRIVATE "-Wl,--no-as-needed" ${CARDINAL_LIBS} "-Wl,--as-needed")
292293
endif()
293294

‎cmake/libs/cardinal/v1/CMakeLists.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ project(knowhere CXX C)
33

44
# Use short SHA1 as version
55
# currently check out a commit for cardinal v1. TODO: need a tag here
6-
set(CARDINAL_VERSION v2.5.107)
6+
set(CARDINAL_VERSION v2.5.108)
77
set(CARDINAL_REPO_URL "https://github.com/zilliztech/cardinal.git")
88

99
set(CARDINAL_ROOT "${KNOWHERE_THRID_ROOT}/cardinalv1")

0 commit comments

Comments
 (0)