Skip to content

Nightly

Nightly #174

Workflow file for this run

name: Nightly
on:
schedule:
# Run at 2am UTC every day
- cron: '0 2 * * *'
workflow_dispatch:
inputs:
skip_tests:
description: 'Skip test jobs (build only)'
required: false
type: boolean
default: false
concurrency:
group: nightly-${{ github.ref_name }}
cancel-in-progress: true
env:
CARGO_TERM_COLOR: always
jobs:
# ============================================================================
# Lint (runs on all PRs, quick feedback)
# ============================================================================
lint:
name: Lint
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Setup workspace
run: |
chmod +x scripts/setup-workspace.sh
./scripts/setup-workspace.sh
- name: Setup Rust
uses: dtolnay/rust-toolchain@stable
- name: Setup sccache
uses: mozilla-actions/sccache-action@v0.0.6
- name: Enable sccache (if available)
run: |
if SCCACHE_GHA_ENABLED=true sccache --start-server; then
echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV
echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV
else
echo "sccache unavailable; proceeding without cache."
fi
- name: Rust cache
uses: Swatinem/rust-cache@v2
with:
workspaces: |
src-tauri
biovault/cli
syftbox-sdk
- name: Setup Node.js
uses: actions/setup-node@v4
with:
node-version: '20'
cache: 'npm'
cache-dependency-path: package-lock.json
- name: Install dependencies
run: npm ci
- name: Run lint checks
run: ./lint.sh --check
# ============================================================================
# Tests - Platform-specific with Docker support consideration
# ============================================================================
test-pipelines:
name: Pipeline Tests (${{ matrix.platform }})
needs: [lint]
if: ${{ !inputs.skip_tests }}
runs-on: ${{ matrix.runner }}
strategy:
fail-fast: false
matrix:
include:
# macOS ARM64 - full tests with Docker
- platform: macos-arm64
runner: macos-14
has_docker: true
# Linux x86_64 - full tests with Docker
- platform: linux-x86
runner: ubuntu-latest
has_docker: true
# Windows - skip Docker-dependent tests
# Note: Windows runners don't support Linux containers
# Pipeline tests require Nextflow which needs Docker
# - platform: windows-x86
# runner: windows-latest
# has_docker: false
steps:
- uses: actions/checkout@v4
- name: Setup workspace
run: |
chmod +x scripts/setup-workspace.sh
./scripts/setup-workspace.sh
- name: Setup Rust
uses: dtolnay/rust-toolchain@stable
- name: Setup sccache
uses: mozilla-actions/sccache-action@v0.0.6
- name: Enable sccache (if available)
run: |
if SCCACHE_GHA_ENABLED=true sccache --start-server; then
echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV
echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV
else
echo "sccache unavailable; proceeding without cache."
fi
- name: Rust cache
uses: Swatinem/rust-cache@v2
with:
workspaces: |
src-tauri
biovault/cli
syftbox-sdk
- name: Setup Node.js
uses: actions/setup-node@v4
with:
node-version: '20'
cache: 'npm'
cache-dependency-path: package-lock.json
- name: Setup Bun
uses: oven-sh/setup-bun@v1
with:
bun-version: '1.1.38'
- name: Cache Bun downloads
uses: actions/cache@v4
with:
path: |
~/.bun/install/cache
~/.cache/bun
~/Library/Caches/bun
~/AppData/Local/bun/install/cache
key: bun-${{ runner.os }}-${{ hashFiles('bun.lock') }}
- name: Setup Go (syftbox)
uses: actions/setup-go@v5
with:
go-version-file: syftbox/go.mod
cache: true
cache-dependency-path: syftbox/go.sum
- name: Setup Python
uses: actions/setup-python@v5
with:
python-version: '3.11'
- name: Install protoc
uses: arduino/setup-protoc@v2
with:
repo-token: ${{ secrets.GITHUB_TOKEN }}
# Linux-specific dependencies
- name: Install Linux dependencies
if: runner.os == 'Linux'
run: |
sudo apt-get update
sudo apt-get install -y libwebkit2gtk-4.1-dev libappindicator3-dev librsvg2-dev patchelf
- name: Install npm dependencies
run: npm ci
- name: Install bun dependencies
run: bun install --frozen-lockfile
- name: Cache Playwright browsers
id: cache-playwright-pipelines
uses: actions/cache@v4
with:
path: |
~/.cache/ms-playwright
~/AppData/Local/ms-playwright
key: playwright-${{ runner.os }}-${{ hashFiles('package-lock.json', 'bun.lock') }}
- name: Install Playwright browsers
if: steps.cache-playwright-pipelines.outputs.cache-hit != 'true'
run: bunx --bun playwright install --with-deps chromium
# Install biosynth - platform specific
- name: Install biosynth CLI (Linux)
if: runner.os == 'Linux'
run: |
curl -fsSL https://github.com/OpenMined/biosynth/releases/latest/download/bvs-x86_64-unknown-linux-gnu.tar.gz | tar -xz
sudo mv bvs /usr/local/bin/
bvs --version
- name: Install biosynth CLI (macOS ARM64)
if: runner.os == 'macOS' && runner.arch == 'ARM64'
run: |
curl -fsSL https://github.com/OpenMined/biosynth/releases/latest/download/bvs-aarch64-apple-darwin.tar.gz | tar -xz
sudo mv bvs /usr/local/bin/
bvs --version
- name: Install biosynth CLI (macOS x86)
if: runner.os == 'macOS' && runner.arch == 'X64'
run: |
curl -fsSL https://github.com/OpenMined/biosynth/releases/latest/download/bvs-x86_64-apple-darwin.tar.gz | tar -xz
sudo mv bvs /usr/local/bin/
bvs --version
- name: Cache genostats database
id: cache-genostats
uses: actions/cache@v4
with:
path: data/genostats.sqlite
key: genostats-v1
- name: Download genostats database
if: steps.cache-genostats.outputs.cache-hit != 'true'
run: |
mkdir -p data
curl -fsSL -o data/genostats.sqlite \
"https://github.com/OpenMined/biovault-desktop/releases/download/data-v1/genostats.sqlite" || {
echo "Warning: Could not download genostats.sqlite"
}
- name: Cache bundled dependencies
id: cache-bundled-pipelines
uses: actions/cache@v4
with:
path: src-tauri/resources/bundled
key: bundled-${{ runner.os }}-${{ hashFiles('scripts/fetch-bundled-deps.sh') }}
- name: Fetch bundled dependencies
if: steps.cache-bundled-pipelines.outputs.cache-hit != 'true'
run: |
chmod +x scripts/fetch-bundled-deps.sh
./scripts/fetch-bundled-deps.sh
- name: Ensure bundled dependency permissions
run: chmod -R u+rw src-tauri/resources/bundled/ || true
- name: Build syftbox client
run: |
chmod +x scripts/build-syftbox-prod.sh
./scripts/build-syftbox-prod.sh
- name: Build Tauri binary (release)
run: |
cd src-tauri
cargo build --release
- name: Setup Docker (macOS)
if: runner.os == 'macOS'
uses: docker-practice/actions-setup-docker@master
timeout-minutes: 12
- name: Verify Docker
if: matrix.has_docker
run: docker info
- name: Run pipeline solo tests
env:
SKIP_PLAYWRIGHT_INSTALL: "1"
AUTO_REBUILD_TAURI: "0"
CLEANUP_SYNTHETIC: "1"
run: ./test-scenario.sh --pipelines-solo
- name: Upload test artifacts on failure
if: failure()
uses: actions/upload-artifact@v4
with:
name: pipelines-test-results-${{ matrix.platform }}
path: |
logs/
test-results/
artifacts/
test-jupyter:
name: Jupyter Tests (${{ matrix.platform }})
needs: [lint]
if: ${{ !inputs.skip_tests }}
runs-on: ${{ matrix.runner }}
strategy:
fail-fast: false
matrix:
include:
# macOS ARM64
- platform: macos-arm64
runner: macos-14
# Linux x86_64
- platform: linux-x86
runner: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Setup workspace
run: |
chmod +x scripts/setup-workspace.sh
./scripts/setup-workspace.sh
- name: Setup Rust
uses: dtolnay/rust-toolchain@stable
- name: Setup sccache
uses: mozilla-actions/sccache-action@v0.0.6
- name: Enable sccache (if available)
run: |
if SCCACHE_GHA_ENABLED=true sccache --start-server; then
echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV
echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV
else
echo "sccache unavailable; proceeding without cache."
fi
- name: Rust cache
uses: Swatinem/rust-cache@v2
with:
workspaces: |
src-tauri
biovault/cli
syftbox-sdk
- name: Setup Node.js
uses: actions/setup-node@v4
with:
node-version: '20'
cache: 'npm'
cache-dependency-path: package-lock.json
- name: Setup Bun
uses: oven-sh/setup-bun@v1
with:
bun-version: '1.1.38'
- name: Cache Bun downloads
uses: actions/cache@v4
with:
path: |
~/.bun/install/cache
~/.cache/bun
~/Library/Caches/bun
~/AppData/Local/bun/install/cache
key: bun-${{ runner.os }}-${{ hashFiles('bun.lock') }}
- name: Setup Go (syftbox)
uses: actions/setup-go@v5
with:
go-version-file: syftbox/go.mod
cache: true
cache-dependency-path: syftbox/go.sum
- name: Setup Python
uses: actions/setup-python@v5
with:
python-version: '3.11'
- name: Install protoc
uses: arduino/setup-protoc@v2
with:
repo-token: ${{ secrets.GITHUB_TOKEN }}
# Linux-specific dependencies
- name: Install Linux dependencies
if: runner.os == 'Linux'
run: |
sudo apt-get update
sudo apt-get install -y libwebkit2gtk-4.1-dev libappindicator3-dev librsvg2-dev patchelf
- name: Install npm dependencies
run: npm ci
- name: Install bun dependencies
run: bun install --frozen-lockfile
- name: Cache Playwright browsers
id: cache-playwright-jupyter
uses: actions/cache@v4
with:
path: |
~/.cache/ms-playwright
~/AppData/Local/ms-playwright
key: playwright-${{ runner.os }}-${{ hashFiles('package-lock.json', 'bun.lock') }}
- name: Install Playwright browsers
if: steps.cache-playwright-jupyter.outputs.cache-hit != 'true'
run: bunx --bun playwright install --with-deps chromium
- name: Cache bundled dependencies
id: cache-bundled-jupyter
uses: actions/cache@v4
with:
path: src-tauri/resources/bundled
key: bundled-${{ runner.os }}-${{ hashFiles('scripts/fetch-bundled-deps.sh') }}
- name: Fetch bundled dependencies
if: steps.cache-bundled-jupyter.outputs.cache-hit != 'true'
run: |
chmod +x scripts/fetch-bundled-deps.sh
./scripts/fetch-bundled-deps.sh
- name: Ensure bundled dependency permissions
run: chmod -R u+rw src-tauri/resources/bundled/ || true
- name: Build syftbox client
run: |
chmod +x scripts/build-syftbox-prod.sh
./scripts/build-syftbox-prod.sh
- name: Build Tauri binary (release)
run: |
cd src-tauri
cargo build --release
- name: Run jupyter session tests (solo)
env:
SKIP_PLAYWRIGHT_INSTALL: "1"
AUTO_REBUILD_TAURI: "0"
INCLUDE_JUPYTER_TESTS: "1"
run: ./test-scenario.sh --jupyter-session
- name: Upload test artifacts on failure
if: failure()
uses: actions/upload-artifact@v4
with:
name: jupyter-test-results-${{ matrix.platform }}
path: |
logs/
test-results/
artifacts/
test-jupyter-collab:
name: Jupyter Collab Tests (${{ matrix.platform }})
needs: [lint]
if: ${{ !inputs.skip_tests }}
runs-on: ${{ matrix.runner }}
strategy:
fail-fast: false
matrix:
include:
# macOS ARM64
- platform: macos-arm64
runner: macos-14
# Linux x86_64
- platform: linux-x86
runner: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Setup workspace
run: |
chmod +x scripts/setup-workspace.sh
./scripts/setup-workspace.sh
- name: Setup Rust
uses: dtolnay/rust-toolchain@stable
- name: Setup sccache
uses: mozilla-actions/sccache-action@v0.0.6
- name: Enable sccache (if available)
run: |
if SCCACHE_GHA_ENABLED=true sccache --start-server; then
echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV
echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV
else
echo "sccache unavailable; proceeding without cache."
fi
- name: Rust cache
uses: Swatinem/rust-cache@v2
with:
workspaces: |
src-tauri
biovault/cli
syftbox-sdk
- name: Setup Node.js
uses: actions/setup-node@v4
with:
node-version: '20'
cache: 'npm'
cache-dependency-path: package-lock.json
- name: Setup Bun
uses: oven-sh/setup-bun@v1
with:
bun-version: '1.1.38'
- name: Cache Bun downloads
uses: actions/cache@v4
with:
path: |
~/.bun/install/cache
~/.cache/bun
~/Library/Caches/bun
~/AppData/Local/bun/install/cache
key: bun-${{ runner.os }}-${{ hashFiles('bun.lock') }}
- name: Setup Go (syftbox)
uses: actions/setup-go@v5
with:
go-version-file: syftbox/go.mod
cache: true
cache-dependency-path: syftbox/go.sum
- name: Setup Python
uses: actions/setup-python@v5
with:
python-version: '3.11'
- name: Install protoc
uses: arduino/setup-protoc@v2
with:
repo-token: ${{ secrets.GITHUB_TOKEN }}
# Linux-specific dependencies
- name: Install Linux dependencies
if: runner.os == 'Linux'
run: |
sudo apt-get update
sudo apt-get install -y libwebkit2gtk-4.1-dev libappindicator3-dev librsvg2-dev patchelf
- name: Install npm dependencies
run: npm ci
- name: Install bun dependencies
run: bun install --frozen-lockfile
- name: Cache Playwright browsers
id: cache-playwright-jupyter-collab
uses: actions/cache@v4
with:
path: |
~/.cache/ms-playwright
~/AppData/Local/ms-playwright
key: playwright-${{ runner.os }}-${{ hashFiles('package-lock.json', 'bun.lock') }}
- name: Install Playwright browsers
if: steps.cache-playwright-jupyter-collab.outputs.cache-hit != 'true'
run: bunx --bun playwright install --with-deps chromium
- name: Cache bundled dependencies
id: cache-bundled-jupyter-collab
uses: actions/cache@v4
with:
path: src-tauri/resources/bundled
key: bundled-${{ runner.os }}-${{ hashFiles('scripts/fetch-bundled-deps.sh') }}
- name: Fetch bundled dependencies
if: steps.cache-bundled-jupyter-collab.outputs.cache-hit != 'true'
run: |
chmod +x scripts/fetch-bundled-deps.sh
./scripts/fetch-bundled-deps.sh
- name: Ensure bundled dependency permissions
run: chmod -R u+rw src-tauri/resources/bundled/ || true
- name: Build syftbox client
run: |
chmod +x scripts/build-syftbox-prod.sh
./scripts/build-syftbox-prod.sh
- name: Build Tauri binary (release)
run: |
cd src-tauri
cargo build --release
- name: Run jupyter collab tests (2-client)
env:
SKIP_PLAYWRIGHT_INSTALL: "1"
AUTO_REBUILD_TAURI: "0"
INCLUDE_JUPYTER_TESTS: "1"
run: ./test-scenario.sh --jupyter-collab biovault-beaver/notebooks/02-advanced-features.json
- name: Upload test artifacts on failure
if: failure()
uses: actions/upload-artifact@v4
with:
name: jupyter-collab-test-results-${{ matrix.platform }}
path: |
logs/
test-results/
artifacts/
test-multi-client:
name: Multi-Client Tests (${{ matrix.platform }})
needs: [lint]
if: ${{ !inputs.skip_tests }}
runs-on: ${{ matrix.runner }}
strategy:
fail-fast: false
matrix:
include:
# macOS ARM64
- platform: macos-arm64
runner: macos-14
# Linux x86_64
- platform: linux-x86
runner: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Setup workspace
run: |
chmod +x scripts/setup-workspace.sh
./scripts/setup-workspace.sh
- name: Setup Rust
uses: dtolnay/rust-toolchain@stable
- name: Setup sccache
uses: mozilla-actions/sccache-action@v0.0.6
- name: Enable sccache (if available)
run: |
if SCCACHE_GHA_ENABLED=true sccache --start-server; then
echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV
echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV
else
echo "sccache unavailable; proceeding without cache."
fi
- name: Rust cache
uses: Swatinem/rust-cache@v2
with:
workspaces: |
src-tauri
biovault/cli
syftbox-sdk
- name: Setup Node.js
uses: actions/setup-node@v4
with:
node-version: '20'
cache: 'npm'
cache-dependency-path: package-lock.json
- name: Setup Bun
uses: oven-sh/setup-bun@v1
with:
bun-version: '1.1.38'
- name: Cache Bun downloads
uses: actions/cache@v4
with:
path: |
~/.bun/install/cache
~/.cache/bun
~/Library/Caches/bun
~/AppData/Local/bun/install/cache
key: bun-${{ runner.os }}-${{ hashFiles('bun.lock') }}
- name: Setup Go (syftbox)
uses: actions/setup-go@v5
with:
go-version-file: syftbox/go.mod
cache: true
cache-dependency-path: syftbox/go.sum
- name: Setup Python
uses: actions/setup-python@v5
with:
python-version: '3.11'
- name: Install protoc
uses: arduino/setup-protoc@v2
with:
repo-token: ${{ secrets.GITHUB_TOKEN }}
# Linux-specific dependencies
- name: Install Linux dependencies
if: runner.os == 'Linux'
run: |
sudo apt-get update
sudo apt-get install -y libwebkit2gtk-4.1-dev libappindicator3-dev librsvg2-dev patchelf
- name: Install npm dependencies
run: npm ci
- name: Install bun dependencies
run: bun install --frozen-lockfile
- name: Cache Playwright browsers
id: cache-playwright-multi
uses: actions/cache@v4
with:
path: |
~/.cache/ms-playwright
~/AppData/Local/ms-playwright
key: playwright-${{ runner.os }}-${{ hashFiles('package-lock.json', 'bun.lock') }}
- name: Install Playwright browsers
if: steps.cache-playwright-multi.outputs.cache-hit != 'true'
run: bunx --bun playwright install --with-deps chromium
- name: Cache bundled dependencies
id: cache-bundled-multi
uses: actions/cache@v4
with:
path: src-tauri/resources/bundled
key: bundled-${{ runner.os }}-${{ hashFiles('scripts/fetch-bundled-deps.sh') }}
- name: Fetch bundled dependencies
if: steps.cache-bundled-multi.outputs.cache-hit != 'true'
run: |
chmod +x scripts/fetch-bundled-deps.sh
./scripts/fetch-bundled-deps.sh
- name: Ensure bundled dependency permissions
run: chmod -R u+rw src-tauri/resources/bundled/ || true
- name: Build syftbox client
run: |
chmod +x scripts/build-syftbox-prod.sh
./scripts/build-syftbox-prod.sh
- name: Build Tauri binary (release)
run: |
cd src-tauri
cargo build --release
- name: Run multi-client scenario tests
env:
SKIP_PLAYWRIGHT_INSTALL: "1"
AUTO_REBUILD_TAURI: "0"
run: ./test-scenario.sh --scenario multi
- name: Upload test artifacts on failure
if: failure()
uses: actions/upload-artifact@v4
with:
name: multi-client-test-results-${{ matrix.platform }}
path: |
logs/
test-results/
artifacts/
# ============================================================================
# Summary job
# ============================================================================
nightly-summary:
name: Nightly Summary
needs: [lint, test-pipelines, test-jupyter, test-jupyter-collab, test-multi-client]
if: always()
runs-on: ubuntu-latest
steps:
- name: Check results
run: |
echo "## Nightly Build Summary" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "| Job | Status |" >> $GITHUB_STEP_SUMMARY
echo "|-----|--------|" >> $GITHUB_STEP_SUMMARY
echo "| Lint | ${{ needs.lint.result }} |" >> $GITHUB_STEP_SUMMARY
echo "| Pipeline Tests | ${{ needs.test-pipelines.result }} |" >> $GITHUB_STEP_SUMMARY
echo "| Jupyter Tests (Solo) | ${{ needs.test-jupyter.result }} |" >> $GITHUB_STEP_SUMMARY
echo "| Jupyter Tests (Collab) | ${{ needs.test-jupyter-collab.result }} |" >> $GITHUB_STEP_SUMMARY
echo "| Multi-Client Tests | ${{ needs.test-multi-client.result }} |" >> $GITHUB_STEP_SUMMARY
- name: Fail if any job failed
if: contains(needs.*.result, 'failure')
run: exit 1