Make wheels contain platform and python ABI tags (#3)

* Made wheels contain platform and python ABI tags

* Changed setuptools requirement; ensured bdist_wheel used on Windows; improved Windows build script

* Used 3.13-slim python image for DockerFile.linux

* Ensured Python 3.13 before running demo

* Ensured linux wheels are manylinux wheels

Author: drawbyperpetual

.github/workflows/build.yml

@@ -123,6 +123,11 @@ jobs:
           mkdir -p wheels/linux
           docker cp jecq-wheel-builder:/app/wheels/linux/. wheels/linux/
           docker rm jecq-wheel-builder
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.13'
 
       - name: Run sample demo in clean environment
         run: |

.vscode/settings.json

@@ -6,7 +6,10 @@
     "cSpell.words": [
         "addlicense",
         "atleast",
+        "auditwheel",
+        "bdist",
         "BLAS",
+        "cmdclass",
         "CMPLR",
         "DBLA",
         "DBUILD",

Dockerfile.linux

@@ -1,4 +1,5 @@
 FROM ubuntu:22.04
+FROM python:3.13-slim
 
 ARG BUILD_TYPE=Debug
 

build.ps1

@@ -65,6 +65,7 @@ Write-Output "Setting up virtual environment..."
 Push-Location .
 python -m venv .venv; Test-Exit-Code
 ./.venv/scripts/activate.ps1; Test-Exit-Code
+python -m pip install --upgrade pip; Test-Exit-Code
 pip install -r ../jecq/python/requirements.txt; Test-Exit-Code
 Pop-Location
 
@@ -149,12 +150,14 @@ Test-Exit-Code
 # Install Python package
 Write-Output "Installing Python package..."
 Push-Location jecq/python
-python setup.py install; Test-Exit-Code
 pip install wheel; Test-Exit-Code
-pip wheel --no-binary jecq .; Test-Exit-Code
+python setup.py bdist_wheel; Test-Exit-Code
+Get-ChildItem dist/jecq*.whl | ForEach-Object { pip install $_.FullName }
+Pop-Location
+
+# Verify installation
 python -c "import jecq;jecq.IndexJecq();jecq.IndexIVFJecq()"; Test-Exit-Code
 python -c "import jecq;assert jecq.IndexJecq.__module__ == 'jecq.swigjecq_avx2'"; Test-Exit-Code
-Pop-Location
 
 # Update wheels
 Write-Output "Updating wheels..."

build.sh

@@ -46,6 +46,7 @@ pushd .
 echo "Setting up virtual environment..."
 python3 -m venv .venv
 source .venv/bin/activate
+python -m pip install --upgrade pip
 pip install -r ../jecq/python/requirements.txt
 popd
 
@@ -87,13 +88,22 @@ fi
 echo "Installing Python package..."
 pushd .
 cd ./jecq/python
-# Make sure wheel is installed
 pip install wheel
-python3 setup.py install
-pip wheel --no-binary jecq .
+python3 setup.py bdist_wheel
+# Repair wheel for Linux compatibility
+pip install auditwheel
+pushd ./dist
+for f in jecq*.whl; do
+  auditwheel repair "$f" -w . && rm "$f"
+done
+popd
+# Install wheels
+pip install dist/jecq*.whl
+popd
+
+# Verify installation
 python3 -c "import jecq;jecq.IndexJecq();jecq.IndexIVFJecq()"
 python3 -c "import jecq;assert jecq.IndexJecq.__module__ == 'jecq.swigjecq_avx2'"
-popd
 
 # Update wheels
 echo "Updating wheels..."

jecq/python/requirements.txt

@@ -1,5 +1,5 @@
 faiss-cpu
 numpy>=2
-setuptools
+setuptools>=80
 packaging
 swig

jecq/python/setup.py

@@ -23,7 +23,8 @@
 import platform
 import shutil
 
-from setuptools import setup
+from setuptools import setup, Extension
+from setuptools.command.build_ext import build_ext as _build_ext
 
 # make the jecq python package dir
 shutil.rmtree("jecq", ignore_errors=True)
@@ -32,23 +33,22 @@
 shutil.copyfile("loader.py", "jecq/loader.py")
 shutil.copyfile("class_wrappers.py", "jecq/class_wrappers.py")
 
-ext = ".pyd" if platform.system() == "Windows" else ".so"
+is_windows = platform.system() == "Windows"
 
+ext = ".pyd" if is_windows else ".so"
 build_type = os.environ.get("JECQ_BUILD_TYPE", "RelWithDebInfo")
-prefix = f"{build_type}/" * (platform.system() == "Windows")
+prefix = f"{build_type}/" * is_windows
 
 swigjecq_generic_lib = f"{prefix}_swigjecq{ext}"
 swigjecq_avx2_lib = f"{prefix}_swigjecq_avx2{ext}"
 swigjecq_avx512_lib = f"{prefix}_swigjecq_avx512{ext}"
 swigjecq_avx512_spr_lib = f"{prefix}_swigjecq_avx512_spr{ext}"
-callbacks_lib = f"{prefix}libfaiss_python_callbacks{ext}"
 swigjecq_sve_lib = f"{prefix}_swigjecq_sve{ext}"
 
 found_swigjecq_generic = os.path.exists(swigjecq_generic_lib)
 found_swigjecq_avx2 = os.path.exists(swigjecq_avx2_lib)
 found_swigjecq_avx512 = os.path.exists(swigjecq_avx512_lib)
 found_swigjecq_avx512_spr = os.path.exists(swigjecq_avx512_spr_lib)
-found_callbacks = os.path.exists(callbacks_lib)
 found_swigjecq_sve = os.path.exists(swigjecq_sve_lib)
 
 assert (
@@ -64,50 +64,93 @@
     f"Jecq may not be compiled yet."
 )
 
+libs = []
+
 if found_swigjecq_generic:
     print(f"Copying {swigjecq_generic_lib}")
     shutil.copyfile("swigjecq.py", "jecq/swigjecq.py")
     shutil.copyfile(swigjecq_generic_lib, f"jecq/_swigjecq{ext}")
+    libs.append("_swigjecq")
 
 if found_swigjecq_avx2:
     print(f"Copying {swigjecq_avx2_lib}")
     shutil.copyfile("swigjecq_avx2.py", "jecq/swigjecq_avx2.py")
     shutil.copyfile(swigjecq_avx2_lib, f"jecq/_swigjecq_avx2{ext}")
+    libs.append("_swigjecq_avx2")
 
 if found_swigjecq_avx512:
     print(f"Copying {swigjecq_avx512_lib}")
     shutil.copyfile("swigjecq_avx512.py", "jecq/swigjecq_avx512.py")
     shutil.copyfile(swigjecq_avx512_lib, f"jecq/_swigjecq_avx512{ext}")
+    libs.append("_swigjecq_avx512")
+
 
 if found_swigjecq_avx512_spr:
     print(f"Copying {swigjecq_avx512_spr_lib}")
     shutil.copyfile("swigjecq_avx512_spr.py", "jecq/swigjecq_avx512_spr.py")
     shutil.copyfile(swigjecq_avx512_spr_lib, f"jecq/_swigjecq_avx512_spr{ext}")
+    libs.append("_swigjecq_avx512_spr")
 
-if found_callbacks:
-    print(f"Copying {callbacks_lib}")
-    shutil.copyfile(callbacks_lib, f"jecq/{callbacks_lib}")
 
 if found_swigjecq_sve:
     print(f"Copying {swigjecq_sve_lib}")
     shutil.copyfile("swigjecq_sve.py", "jecq/swigjecq_sve.py")
     shutil.copyfile(swigjecq_sve_lib, f"jecq/_swigjecq_sve{ext}")
+    libs.append("_swigjecq_sve")
+
+ext_modules = [Extension(f"jecq.{mod}", sources=[]) for mod in libs]
+
+
+# 2) CopyBuildExt just copies the .so into the build directory without compiling
+class CopyBuildExt(_build_ext):
+    def run(self):
+        # skip the normal compiler run
+        for extension in self.extensions:
+            self.build_extension(extension)
+
+    def build_extension(self, extension):
+        name = extension.name.split(".", 1)[1]  # e.g. "_swigjecq"
+        src = f"{prefix}{name}{ext}"
+        if not os.path.exists(src):
+            raise FileNotFoundError(f"Binary output '{src}' not found")
+        dst = self.get_ext_fullpath(extension.name)
+        self.mkpath(os.path.dirname(dst))
+        shutil.copyfile(src, dst)
+        print(f"Copied {src} to {dst}")
+
 
 long_description = """
 Jecq is a library for efficient similarity search based on the Faiss library.
 """
+
+try:
+    from wheel.bdist_wheel import bdist_wheel as _bdist_wheel
+
+    """Custom bdist_wheel to ensure that the root is not pure Python."""
+
+    class bdist_wheel(_bdist_wheel):
+        def finalize_options(self):
+            _bdist_wheel.finalize_options(self)
+            self.root_is_pure = False
+
+except ImportError:
+    print("Not using custom bdist_wheel; wheel package not installed")
+
 setup(
     name="jecq",
     version="0.0.1",
     description="A Faiss-based library for efficient similarity search "
     "and clustering of dense vectors",
     long_description=long_description,
-    license="TODO",
+    url="https://github.com/JaneaSystems/jecq",
+    license="MIT",
     keywords="search nearest neighbors",
     install_requires=["numpy", "packaging", "faiss-cpu"],
     packages=["jecq"],
     package_data={
-        "jecq": ["*.so", "*.pyd"],
+        "jecq": ["*.pyd"] if is_windows else [],
     },
     zip_safe=False,
+    ext_modules=[] if is_windows else ext_modules,
+    cmdclass={"bdist_wheel": bdist_wheel, "build_ext": CopyBuildExt},
 )

requirements.linux

@@ -7,4 +7,5 @@ cmake
 build-essential
 autoconf
 automake
-libtool
+libtool
+patchelf