Skip to content

Commit 59c5f42

Browse files
Michael Norrisfacebook-github-bot
Michael Norris
authored and
facebook-github-bot
committed
Index serialization backward compatibility test (#4706)
Summary: Test backwards compatibility by checking the conda build of the a Faiss release vs current code (built by cmake). This will run as a Github Action. Differential Revision: D88085793
1 parent 8cff802 commit 59c5f42

8 files changed

Lines changed: 1027 additions & 106 deletions

File tree

‎.github/workflows/build-pull-request.yml‎

Lines changed: 109 additions & 106 deletions
Original file line numberDiff line numberDiff line change
@@ -38,109 +38,112 @@ jobs:
3838
uses: actions/checkout@v4
3939
- name: Build and Test (cmake)
4040
uses: ./.github/actions/build_cmake
41-
linux-x86_64-AVX2-cmake:
42-
name: Linux x86_64 AVX2 (cmake)
43-
needs: linux-x86_64-cmake
44-
runs-on: ubuntu-latest
45-
steps:
46-
- name: Checkout
47-
uses: actions/checkout@v4
48-
- name: Build and Test (cmake)
49-
uses: ./.github/actions/build_cmake
50-
with:
51-
opt_level: avx2
52-
linux-x86_64-AVX512-cmake:
53-
name: Linux x86_64 AVX512 (cmake)
54-
needs: linux-x86_64-cmake
55-
runs-on: faiss-aws-m7i.large
56-
steps:
57-
- name: Checkout
58-
uses: actions/checkout@v4
59-
- name: Build and Test (cmake)
60-
uses: ./.github/actions/build_cmake
61-
with:
62-
opt_level: avx512
63-
linux-x86_64-AVX512_SPR-cmake:
64-
name: Linux x86_64 AVX512_SPR (cmake)
65-
needs: linux-x86_64-cmake
66-
runs-on: faiss-aws-m7i.large
67-
steps:
68-
- name: Checkout
69-
uses: actions/checkout@v4
70-
- name: Build and Test (cmake)
71-
uses: ./.github/actions/build_cmake
72-
with:
73-
opt_level: avx512_spr
74-
linux-x86_64-GPU-cmake:
75-
name: Linux x86_64 GPU (cmake)
76-
needs: linux-x86_64-cmake
77-
runs-on: 4-core-ubuntu-gpu-t4
78-
steps:
79-
- name: Checkout
80-
uses: actions/checkout@v4
81-
- name: Build and Test (cmake)
82-
uses: ./.github/actions/build_cmake
83-
with:
84-
gpu: ON
85-
linux-x86_64-GPU-w-CUVS-cmake:
86-
name: Linux x86_64 GPU w/ cuVS (cmake)
87-
needs: linux-x86_64-cmake
88-
runs-on: 4-core-ubuntu-gpu-t4
89-
steps:
90-
- name: Checkout
91-
uses: actions/checkout@v4
92-
- name: Build and Test (cmake)
93-
uses: ./.github/actions/build_cmake
94-
with:
95-
gpu: ON
96-
cuvs: ON
97-
linux-arm64-SVE-cmake:
98-
name: Linux arm64 SVE (cmake)
99-
needs: linux-x86_64-cmake
100-
runs-on: faiss-aws-r8g.large
101-
steps:
102-
- name: Checkout
103-
uses: actions/checkout@v4
104-
- name: Build and Test (cmake)
105-
uses: ./.github/actions/build_cmake
106-
with:
107-
opt_level: sve
108-
env:
109-
# Context: https://github.com/facebookresearch/faiss/wiki/Troubleshooting#surprising-faiss-openmp-and-openblas-interaction
110-
OPENBLAS_NUM_THREADS: '1'
111-
linux-x86_64-conda:
112-
name: Linux x86_64 (conda)
113-
needs: linux-x86_64-cmake
114-
runs-on: ubuntu-latest
115-
steps:
116-
- name: Checkout
117-
uses: actions/checkout@v4
118-
with:
119-
fetch-depth: 0
120-
fetch-tags: true
121-
- name: Build and Package (conda)
122-
uses: ./.github/actions/build_conda
123-
windows-x86_64-conda:
124-
name: Windows x86_64 (conda)
125-
needs: linux-x86_64-cmake
126-
runs-on: windows-2022
127-
steps:
128-
- name: Checkout
129-
uses: actions/checkout@v4
130-
with:
131-
fetch-depth: 0
132-
fetch-tags: true
133-
- name: Build and Package (conda)
134-
uses: ./.github/actions/build_conda
135-
linux-arm64-conda:
136-
name: Linux arm64 (conda)
137-
needs: linux-x86_64-cmake
138-
runs-on: 2-core-ubuntu-arm
139-
steps:
140-
- name: Checkout
141-
uses: actions/checkout@v4
142-
with:
143-
fetch-depth: 0
144-
fetch-tags: true
145-
- name: Build and Package (conda)
146-
uses: ./.github/actions/build_conda
41+
cross-build-file-io:
42+
name: Cross-Build File I/O
43+
uses: ./.github/workflows/cross-build-file-io.yml
44+
# linux-x86_64-AVX2-cmake:
45+
# name: Linux x86_64 AVX2 (cmake)
46+
# needs: linux-x86_64-cmake
47+
# runs-on: ubuntu-latest
48+
# steps:
49+
# - name: Checkout
50+
# uses: actions/checkout@v4
51+
# - name: Build and Test (cmake)
52+
# uses: ./.github/actions/build_cmake
53+
# with:
54+
# opt_level: avx2
55+
# linux-x86_64-AVX512-cmake:
56+
# name: Linux x86_64 AVX512 (cmake)
57+
# needs: linux-x86_64-cmake
58+
# runs-on: faiss-aws-m7i.large
59+
# steps:
60+
# - name: Checkout
61+
# uses: actions/checkout@v4
62+
# - name: Build and Test (cmake)
63+
# uses: ./.github/actions/build_cmake
64+
# with:
65+
# opt_level: avx512
66+
# linux-x86_64-AVX512_SPR-cmake:
67+
# name: Linux x86_64 AVX512_SPR (cmake)
68+
# needs: linux-x86_64-cmake
69+
# runs-on: faiss-aws-m7i.large
70+
# steps:
71+
# - name: Checkout
72+
# uses: actions/checkout@v4
73+
# - name: Build and Test (cmake)
74+
# uses: ./.github/actions/build_cmake
75+
# with:
76+
# opt_level: avx512_spr
77+
# linux-x86_64-GPU-cmake:
78+
# name: Linux x86_64 GPU (cmake)
79+
# needs: linux-x86_64-cmake
80+
# runs-on: 4-core-ubuntu-gpu-t4
81+
# steps:
82+
# - name: Checkout
83+
# uses: actions/checkout@v4
84+
# - name: Build and Test (cmake)
85+
# uses: ./.github/actions/build_cmake
86+
# with:
87+
# gpu: ON
88+
# linux-x86_64-GPU-w-CUVS-cmake:
89+
# name: Linux x86_64 GPU w/ cuVS (cmake)
90+
# needs: linux-x86_64-cmake
91+
# runs-on: 4-core-ubuntu-gpu-t4
92+
# steps:
93+
# - name: Checkout
94+
# uses: actions/checkout@v4
95+
# - name: Build and Test (cmake)
96+
# uses: ./.github/actions/build_cmake
97+
# with:
98+
# gpu: ON
99+
# cuvs: ON
100+
# linux-arm64-SVE-cmake:
101+
# name: Linux arm64 SVE (cmake)
102+
# needs: linux-x86_64-cmake
103+
# runs-on: faiss-aws-r8g.large
104+
# steps:
105+
# - name: Checkout
106+
# uses: actions/checkout@v4
107+
# - name: Build and Test (cmake)
108+
# uses: ./.github/actions/build_cmake
109+
# with:
110+
# opt_level: sve
111+
# env:
112+
# # Context: https://github.com/facebookresearch/faiss/wiki/Troubleshooting#surprising-faiss-openmp-and-openblas-interaction
113+
# OPENBLAS_NUM_THREADS: '1'
114+
# linux-x86_64-conda:
115+
# name: Linux x86_64 (conda)
116+
# needs: linux-x86_64-cmake
117+
# runs-on: ubuntu-latest
118+
# steps:
119+
# - name: Checkout
120+
# uses: actions/checkout@v4
121+
# with:
122+
# fetch-depth: 0
123+
# fetch-tags: true
124+
# - name: Build and Package (conda)
125+
# uses: ./.github/actions/build_conda
126+
# windows-x86_64-conda:
127+
# name: Windows x86_64 (conda)
128+
# needs: linux-x86_64-cmake
129+
# runs-on: windows-2022
130+
# steps:
131+
# - name: Checkout
132+
# uses: actions/checkout@v4
133+
# with:
134+
# fetch-depth: 0
135+
# fetch-tags: true
136+
# - name: Build and Package (conda)
137+
# uses: ./.github/actions/build_conda
138+
# linux-arm64-conda:
139+
# name: Linux arm64 (conda)
140+
# needs: linux-x86_64-cmake
141+
# runs-on: 2-core-ubuntu-arm
142+
# steps:
143+
# - name: Checkout
144+
# uses: actions/checkout@v4
145+
# with:
146+
# fetch-depth: 0
147+
# fetch-tags: true
148+
# - name: Build and Package (conda)
149+
# uses: ./.github/actions/build_conda

‎.github/workflows/cross-build-file-io.yml‎

Lines changed: 146 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,146 @@
1+
name: Cross-Build File I/O Test
2+
3+
on:
4+
workflow_call:
5+
6+
env:
7+
OMP_NUM_THREADS: '10'
8+
MKL_THREADING_LAYER: GNU
9+
10+
jobs:
11+
# Scenario 1: CMake writes files, then Conda reads them
12+
cmake-write-conda-read:
13+
name: CMake Write -> Conda Read
14+
runs-on: ubuntu-latest
15+
env:
16+
# run_id does not change if we re-run the job, so this is safe for
17+
# the partway-failure-rerun scenario.
18+
SHARED_DATA_DIR: /tmp/faiss-cross-build-${{ github.run_id }}-cmake-write
19+
steps:
20+
- name: Checkout
21+
uses: actions/checkout@v4
22+
23+
# Step 1: Build with CMake and write files
24+
- name: Build with CMake
25+
uses: ./.github/actions/build_cmake
26+
27+
- name: Create shared data directory
28+
shell: bash
29+
run: |
30+
mkdir -p ${{ env.SHARED_DATA_DIR }}
31+
chmod 777 ${{ env.SHARED_DATA_DIR }}
32+
33+
- name: Run CMake writer (write Faiss index and metadata)
34+
shell: bash
35+
run: |
36+
eval "$(conda shell.bash hook)"
37+
conda activate
38+
$CONDA/bin/python tests/cross_build_io/cmake_writer.py ${{ env.SHARED_DATA_DIR }}
39+
40+
- name: Verify files were written
41+
shell: bash
42+
run: |
43+
echo "Files created by CMake build:"
44+
ls -lh ${{ env.SHARED_DATA_DIR }}
45+
46+
# Step 2: Install conda faiss-cpu package and read files
47+
- name: Setup miniconda
48+
uses: conda-incubator/setup-miniconda@v3
49+
with:
50+
python-version: '3.11'
51+
miniforge-version: latest
52+
53+
- name: Install faiss-cpu from pytorch channel
54+
shell: bash
55+
run: |
56+
eval "$(conda shell.bash hook)"
57+
conda activate
58+
conda install -y -c pytorch faiss-cpu=1.12.0
59+
conda list | grep faiss
60+
61+
- name: Run Conda reader (read Faiss index and verify)
62+
shell: bash
63+
run: |
64+
eval "$(conda shell.bash hook)"
65+
conda activate
66+
python tests/cross_build_io/conda_reader.py ${{ env.SHARED_DATA_DIR }}
67+
68+
- name: Upload artifacts from cmake->conda test
69+
if: always()
70+
uses: actions/upload-artifact@v4
71+
with:
72+
name: cmake-write-conda-read-data
73+
path: ${{ env.SHARED_DATA_DIR }}
74+
75+
# Scenario 2: Conda writes files, then CMake reads them
76+
conda-write-cmake-read:
77+
name: Conda Write -> CMake Read
78+
runs-on: ubuntu-latest
79+
env:
80+
SHARED_DATA_DIR: /tmp/faiss-cross-build-${{ github.run_id }}-conda-write
81+
steps:
82+
- name: Checkout
83+
uses: actions/checkout@v4
84+
with:
85+
fetch-depth: 0
86+
fetch-tags: true
87+
88+
# Step 1: Install conda faiss-cpu package and write files
89+
- name: Setup miniconda
90+
uses: conda-incubator/setup-miniconda@v3
91+
with:
92+
python-version: '3.11'
93+
miniforge-version: latest
94+
95+
- name: Install faiss-cpu from pytorch channel
96+
shell: bash
97+
run: |
98+
eval "$(conda shell.bash hook)"
99+
conda activate
100+
conda install -y -c pytorch faiss-cpu=1.12.0
101+
conda list | grep faiss
102+
103+
- name: Create shared data directory
104+
shell: bash
105+
run: |
106+
mkdir -p ${{ env.SHARED_DATA_DIR }}
107+
chmod 777 ${{ env.SHARED_DATA_DIR }}
108+
109+
- name: Run Conda writer (write Faiss index and metadata)
110+
shell: bash
111+
run: |
112+
eval "$(conda shell.bash hook)"
113+
conda activate
114+
python tests/cross_build_io/conda_writer.py ${{ env.SHARED_DATA_DIR }}
115+
116+
- name: Verify files were written
117+
shell: bash
118+
run: |
119+
echo "Files created by Conda build:"
120+
ls -lh ${{ env.SHARED_DATA_DIR }}
121+
122+
# Step 2: Build with CMake and read files
123+
- name: Clean conda artifacts
124+
shell: bash
125+
run: |
126+
# Uninstall conda-built faiss to avoid conflicts
127+
eval "$(conda shell.bash hook)"
128+
conda activate
129+
conda uninstall -y faiss || true
130+
131+
- name: Build with CMake
132+
uses: ./.github/actions/build_cmake
133+
134+
- name: Run CMake reader (read Faiss index and verify)
135+
shell: bash
136+
run: |
137+
eval "$(conda shell.bash hook)"
138+
conda activate
139+
$CONDA/bin/python tests/cross_build_io/cmake_reader.py ${{ env.SHARED_DATA_DIR }}
140+
141+
- name: Upload artifacts from conda->cmake test
142+
if: always()
143+
uses: actions/upload-artifact@v4
144+
with:
145+
name: conda-write-cmake-read-data
146+
path: ${{ env.SHARED_DATA_DIR }}

‎tests/cross_build_io/__init__.py‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
# Copyright (c) Meta Platforms, Inc. and affiliates.
2+
#
3+
# This source code is licensed under the MIT license found in the
4+
# LICENSE file in the root directory of this source tree.
5+
6+
"""
7+
Cross-build file I/O testing package for Faiss.
8+
9+
This package contains test scripts for validating file I/O compatibility
10+
between cmake-built and conda-built Faiss installations.
11+
"""

0 commit comments

Comments
 (0)