Feature/aarch wheels

.github/workflows/CI.yml

@@ -15,49 +15,139 @@ on:
   pull_request:
   workflow_dispatch:
 
-permissions:
-  contents: read
 
 jobs:
   linux:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        target: [x86_64]
+        target: [x86_64, "aarch64", "armv7"]
+        manylinux: [auto]
+        # include:
+        #   # musllinux
+        #   - os: ubuntu
+        #     target: x86_64-unknown-linux-musl
+        #     manylinux: musllinux_1_2
+        #   - os: ubuntu
+        #     target: aarch64-unknown-linux-musl
+        #     manylinux: musllinux_1_2
+        #   - os: ubuntu
+        #     target: armv7-unknown-linux-musleabihf
+        #     manylinux: musllinux_1_2
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+        with:
+          submodules: 'recursive' # Fetch Git submodules recursively
+
+    # Initialize and update submodules (optional, if submodules were not initialized during checkout)
+      - name: Update submodules
+        run: git submodule update --init --recursive
       - uses: actions/setup-python@v4
         with:
-          python-version: '3.10'
+          python-version: '3.12'
       - name: Build wheels
         uses: PyO3/maturin-action@v1
         with:
           target: ${{ matrix.target }}
-          args: --release --out dist --find-interpreter
+          args: --release --out dist -i 3.8 3.9 3.10 3.11 3.12 pypy3.8 pypy3.9 pypy3.10
           sccache: 'true'
-          manylinux: auto
+          manylinux: ${{ matrix.manylinux || 'auto' }}
+      - name: Install and test built x86_64 wheel
+        if: matrix.target == 'x86_64'
+        run: |
+          pip install mappy-rs  --no-index --find-links dist --force-reinstall
+          pip install pytest
+          pytest -sv
+      # Test musl wheels fro x86
+      - name: Install and test build x86_64 musl wheel
+        if: matrix.target == 'x86_64-unknown-linux-musl'
+        uses: addnab/docker-run-action@v3
+        with:
+          image: alpine:latest
+          options: -v ${{ github.workspace }}:/io -w /io
+          run: |
+            apk add py3-pip
+            pip3 install -U pip pytest
+            pip3 install mappy-rs --no-index --find-links /io/dist/ --force-reinstall
+            python3 -m pytest -sv
       - name: Upload wheels
         uses: actions/upload-artifact@v3
         with:
           name: wheels
           path: dist
 
+  test-builds-aarch:
+    name: run test of build on ${{ matrix.target }}-${{ matrix.distro }}
+    needs: [linux]
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        target: [aarch64, armv7]
+        distro: ['ubuntu22.04']
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: get dist artifacts
+        uses: actions/download-artifact@v3
+        with:
+          name: wheels
+          path: dist
+
+      - uses: uraimo/[email protected]
+        name: install & test
+        with:
+          arch: ${{ matrix.target }}
+          distro: ${{ matrix.distro }}
+          githubToken: ${{ github.token }}
+          install: |
+            set -x
+            if command -v apt-get &> /dev/null; then
+              echo "installing python & pip with apt-get..."
+              apt-get update
+              apt-get install -y --no-install-recommends python3 python3-pip python3-venv
+              pip install -U pip pytest
+            else
+              echo "installing python & pip with apk..."
+              apk update
+              apk add py3-pip
+              pip3 install -U pip pytest
+
+            fi
+          run: |
+            python3 -m pip install mappy-rs --no-index --find-links dist --force-reinstall
+            python3 -m pytest -sv
+
   macos:
-    runs-on: macos-latest
+    runs-on: macos-13
     strategy:
       matrix:
         target: [x86_64, aarch64]
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+        with:
+          submodules: 'recursive' # Fetch Git submodules recursively
+
+    # Initialize and update submodules (optional, if submodules were not initialized during checkout)
+      - name: Update submodules
+        run: git submodule update --init --recursive
       - uses: actions/setup-python@v4
         with:
-          python-version: '3.10'
+          python-version: '3.11'
       - name: Build wheels
         uses: PyO3/maturin-action@v1
         with:
           target: ${{ matrix.target }}
-          args: --release --out dist --find-interpreter
+          args: --release --out dist -i 3.8 3.9 3.10 3.11 3.12 pypy3.8 pypy3.9 pypy3.10
           sccache: 'true'
+      - name: Test built wheels
+        if: matrix.target == 'x86_64'
+        run: |
+          pip install mappy-rs --no-index --find-links dist --force-reinstall
+          pip install pytest
+          pytest -sv
       - name: Upload wheels
         uses: actions/upload-artifact@v3
         with:
@@ -67,7 +157,13 @@ jobs:
   sdist:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+        with:
+          submodules: 'recursive' # Fetch Git submodules recursively
+
+    # Initialize and update submodules (optional, if submodules were not initialized during checkout)
+      - name: Update submodules
+        run: git submodule update --init --recursive
       - name: Build sdist
         uses: PyO3/maturin-action@v1
         with:
@@ -83,7 +179,7 @@ jobs:
     name: Release
     runs-on: ubuntu-latest
     if: "startsWith(github.ref, 'refs/tags/')"
-    needs: [linux, macos, sdist]
+    needs: [linux, test-builds-aarch, macos, sdist]
     permissions:
       # Used to upload release artifacts
       contents: write

.github/workflows/check.yml

@@ -3,21 +3,28 @@ name: check
 
 on:
   push:
-    branches: [main]
+    branches: [main, feature/*]
+    tags:
+      - '*'
   pull_request:
   workflow_dispatch:
   schedule:
     - cron:  '0 0 * * SUN'
 
 env:
-  PYTHON_LATEST: "3.11"
+  PYTHON_LATEST: "3.12"
 
 jobs:
   pre-commit:
     name: "pre-commit"
     runs-on: "ubuntu-latest"
     steps:
-      - uses: "actions/checkout@v3"
+      - uses: "actions/checkout@v4"
+        with:
+          submodules: 'recursive' # Fetch Git submodules recursively
+    # Initialize and update submodules (optional, if submodules were not initialized during checkout)
+      - name: Update submodules
+        run: git submodule update --init --recursive
       - uses: "actions/setup-python@v4"
         with:
           python-version: ${{env.PYTHON_LATEST}}
@@ -33,7 +40,13 @@ jobs:
       matrix:
         toolchain: [stable, beta, nightly]
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+        with:
+          submodules: 'recursive' # Fetch Git submodules recursively
+
+    # Initialize and update submodules (optional, if submodules were not initialized during checkout)
+      - name: Update submodules
+        run: git submodule update --init --recursive
       - uses: "dtolnay/rust-toolchain@master"
         with:
           toolchain: ${{ matrix.toolchain }}
@@ -46,7 +59,13 @@ jobs:
     runs-on: ubuntu-latest
     needs: clippy
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+        with:
+          submodules: 'recursive' # Fetch Git submodules recursively
+
+    # Initialize and update submodules (optional, if submodules were not initialized during checkout)
+      - name: Update submodules
+        run: git submodule update --init --recursive
       - uses: dtolnay/rust-toolchain@stable
       - run: cargo test --no-default-features
 
@@ -56,10 +75,16 @@ jobs:
     needs: clippy
     strategy:
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
 
     steps:
-      - uses: "actions/checkout@v3"
+      - uses: "actions/checkout@v4"
+        with:
+          submodules: 'recursive' # Fetch Git submodules recursively
+
+    # Initialize and update submodules (optional, if submodules were not initialized during checkout)
+      - name: Update submodules
+        run: git submodule update --init --recursive
       - uses: "actions/setup-python@v4"
         with:
           python-version: "${{ matrix.python-version }}"
@@ -79,10 +104,16 @@ jobs:
     needs: ["cargo-test", "python-test"]
     strategy:
       matrix:
-        msrv: ["1.63.0", "1.64.0"]
+        msrv: ["1.76.0", "1.77.0"]
     name: "ubuntu / ${{ matrix.msrv }}"
     steps:
-      - uses: "actions/checkout@v3"
+      - uses: "actions/checkout@v4"
+        with:
+          submodules: 'recursive' # Fetch Git submodules recursively
+
+    # Initialize and update submodules (optional, if submodules were not initialized during checkout)
+      - name: Update submodules
+        run: git submodule update --init --recursive
       - uses: "dtolnay/rust-toolchain@master"
         with:
           toolchain: "${{ matrix.msrv }}"

.gitignore

@@ -271,3 +271,6 @@ cython_debug/
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 #.idea/
+
+# pytest Benchmark histogram output
+*.svg

.gitmodules

@@ -0,0 +1,4 @@
+[submodule "minimap2-rs"]
+	path = minimap2-rs
+	url = https://github.com/Adoni5/minimap2-rs.git
+	branch = main

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "mappy-rs"
-version = "0.0.7"
+version = "0.0.8-alpha.1"
 edition = "2021"
 authors = ["Rory Munro <[email protected]>"]
 license = "MIT OR Apache-2.0"
@@ -21,13 +21,12 @@ name = "mappy_rs"
 [dependencies]
 libc = "0.2"
 pyo3 = { version = "0.19" }
-minimap2-sys = { version = "0.1.15+minimap2.2.26", features = ["simde"] }
 crossbeam = "0.8.2"
 fnv = "1.0.7"
 ctrlc = {version = "3.4.0", features = ["termination"] }
 itertools = "0.10.5"
-minimap2 = {version = "0.1.15+minimap2.2.26" }
-
+minimap2-sys = { path = "./minimap2-rs/minimap2-sys", features = ["simde"] }
+minimap2 = { path = "./minimap2-rs", features = ["simde"] }
 [features]
 extension-module = ["pyo3/extension-module"]
 default = ["extension-module"]

README.md

@@ -103,6 +103,11 @@ test_benchmark_multi[1]       | 84.8418 (3.16)| 94.0907 (3.13) | 86.7404 (3.09)|
 
 
 # Changelog
+## 0.0.8
+- Updated `map_batch` - now has a max attempts parameter, which defaults to 6. If set to `None`, there is no limit to the number of attempts to add a fastq to the mapping batch.
+
+## 0.0.7
+- Added wheel builds for aarch64 linux and MacOS, and armv7 in the CI/yml.
 
 ## 0.0.6
 - Lowered backoff time for `map_batch` to 50 milliseconds, with 6 attempts. Each attempt will double the previous back off time.

help.py

@@ -0,0 +1,9 @@
+# Test the memory issue, not dropping Aligner on rust
+# side after del without manual gc.collect()
+from mappy_rs import Aligner
+import gc
+
+al = Aligner("/home/adoni5/Documents/Bioinformatics/refs/hg38_no_alts_22.mmi")
+al.enable_threading(8)
+del al
+gc.collect()

pyproject.toml

@@ -16,7 +16,7 @@ classifiers = [
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
 ]
-version = "0.0.7a"
+version = "0.0.8a1"
 license = {text = "BSD 3-Clause License"}
 description = "A multithreaded python wrapper for rust bindings of minimap2."
 authors = [