diff --git a/.github/workflows/bench_perf.yml b/.github/workflows/bench_perf.yml new file mode 100644 index 00000000..345693d8 --- /dev/null +++ b/.github/workflows/bench_perf.yml @@ -0,0 +1,207 @@ +name: Performance Benchmarks + +# Mirrors the mountpoint-s3 benchmark CI approach. +# mountpoint-s3 runs on m5dn.24xlarge (100 Gbps, NVMe local cache). +# For comparable results, run on a high-network instance of equivalent class. +# +# Triggers: +# - Every push to main -> publishes results to gh-pages (historical charts) +# - Manual dispatch -> full 100G benchmarks with pre-populated bucket + +on: + push: + branches: [main] + pull_request: + branches: [main] + workflow_dispatch: + inputs: + bucket: + description: "Pre-populated bucket for big file benchmarks (e.g. XciD/hf-bench)" + required: true + default: "XciD/hf-bench" + job_filter: + description: "Job name filter (empty = all jobs including 100G)" + required: false + default: "" + iterations: + description: "Number of iterations per job" + required: false + default: "3" + categories: + description: "Comma-separated benchmark categories" + required: false + default: "read" + +jobs: + bench: + name: Throughput + runs-on: + group: hf-mount-ci-m5dn-24xlarge + timeout-minutes: 120 + permissions: + contents: write + pull-requests: write + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} + steps: + - uses: actions/checkout@v4 + + - uses: dtolnay/rust-toolchain@stable + + - uses: Swatinem/rust-cache@v2 + + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y fuse3 libfuse3-dev fio jq + echo 'user_allow_other' | sudo tee -a /etc/fuse.conf + + - name: Build + run: cargo build --release + + - name: Run throughput benchmarks + run: | + if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then + # Full benchmarks with pre-populated bucket + HF_BENCH_BUCKET="${{ inputs.bucket }}" \ + HF_JOB_NAME_FILTER="${{ inputs.job_filter }}" \ + iterations=${{ inputs.iterations }} \ + HF_CATEGORIES="${{ inputs.categories }}" \ + ./scripts/fs_bench.sh 2>&1 | tee bench.log + else + # Push to main: small files only to keep runtime reasonable + HF_JOB_NAME_FILTER=small iterations=3 \ + HF_CATEGORIES=read,write,mix \ + ./scripts/fs_bench.sh 2>&1 | tee bench.log + fi + + - name: Publish throughput results to gh-pages + uses: benchmark-action/github-action-benchmark@v1 + with: + name: Throughput (MiB/s) + tool: customBiggerIsBetter + output-file-path: results/fuse/output.json + benchmark-data-dir-path: dev/bench + github-token: ${{ secrets.GITHUB_TOKEN }} + auto-push: ${{ github.event_name == 'push' }} + comment-on-alert: true + alert-threshold: 150% + fail-on-alert: false + summary-always: true + + - name: Post results summary + if: always() + run: | + echo "## Throughput Benchmark Results" >> "$GITHUB_STEP_SUMMARY" + echo "" >> "$GITHUB_STEP_SUMMARY" + echo "| Job | Throughput |" >> "$GITHUB_STEP_SUMMARY" + echo "|-----|-----------|" >> "$GITHUB_STEP_SUMMARY" + jq -r '.[] | "| \(.name) | \(.value | round) \(.unit) |"' results/fuse/output.json >> "$GITHUB_STEP_SUMMARY" 2>/dev/null || echo "| (no results) | |" >> "$GITHUB_STEP_SUMMARY" + + - name: Post PR comment + if: github.event_name == 'pull_request' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + RESULTS=$(jq -r '.[] | "| \(.name) | \(.value | round) \(.unit) |"' results/fuse/output.json) + PR="${{ github.event.pull_request.number }}" + MARKER="" + BODY=$(cat <&1 | tee latency.log + + - name: Publish latency results to gh-pages + uses: benchmark-action/github-action-benchmark@v1 + with: + name: Latency - TTFB (ms) + tool: customSmallerIsBetter + output-file-path: results/fuse/output.json + benchmark-data-dir-path: dev/latency_bench + github-token: ${{ secrets.GITHUB_TOKEN }} + auto-push: ${{ github.event_name == 'push' }} + comment-on-alert: true + alert-threshold: 150% + fail-on-alert: false + summary-always: true + + - name: Post results summary + if: always() + run: | + echo "## Latency Benchmark Results (TTFB)" >> "$GITHUB_STEP_SUMMARY" + echo "" >> "$GITHUB_STEP_SUMMARY" + echo "| Job | Latency |" >> "$GITHUB_STEP_SUMMARY" + echo "|-----|---------|" >> "$GITHUB_STEP_SUMMARY" + jq -r '.[] | "| \(.name) | \(.value | . * 100 | round | . / 100) \(.unit) |"' results/fuse/output.json >> "$GITHUB_STEP_SUMMARY" 2>/dev/null || echo "| (no results) | |" >> "$GITHUB_STEP_SUMMARY" + + - name: Post PR comment + if: github.event_name == 'pull_request' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + RESULTS=$(jq -r '.[] | "| \(.name) | \(.value | . * 100 | round | . / 100) \(.unit) |"' results/fuse/output.json) + PR="${{ github.event.pull_request.number }}" + MARKER="" + BODY=$(cat <&1 | tee bench_output.txt - - - name: Post benchmark results - if: github.event_name == 'pull_request' - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - # Strip ANSI codes, then extract only ==== delimited blocks - CLEAN=$(sed 's/\x1b\[[0-9;]*m//g' bench_output.txt) - TABLES=$(echo "$CLEAN" | awk ' - /^=====*$/ { inside=!inside; print; next } - inside { print } - ') - if [ -z "$TABLES" ]; then - echo "No benchmark table found in output" - exit 0 - fi - - PR="${{ github.event.pull_request.number }}" - MARKER="" - BODY="${MARKER} - ## Benchmark Results - - \`\`\` - ${TABLES} - \`\`\`" - - # Find existing comment with our marker - COMMENT_ID=$(gh api "repos/${{ github.repository }}/issues/${PR}/comments" \ - --jq ".[] | select(.body | startswith(\"${MARKER}\")) | .id" | head -1) - - if [ -n "$COMMENT_ID" ]; then - gh api "repos/${{ github.repository }}/issues/comments/${COMMENT_ID}" \ - -X PATCH -f body="$BODY" - echo "Updated existing comment $COMMENT_ID" - else - gh pr comment "$PR" --body "$BODY" - echo "Created new comment" - fi diff --git a/scripts/fio/create/create_files_100.fio b/scripts/fio/create/create_files_100.fio new file mode 100644 index 00000000..c6ce09eb --- /dev/null +++ b/scripts/fio/create/create_files_100.fio @@ -0,0 +1,18 @@ +[global] +create_on_open=1 +nrfiles=10 +ioengine=filecreate +fallocate=none +filesize=4k +openfiles=1 + +[t0] +[t1] +[t2] +[t3] +[t4] +[t5] +[t6] +[t7] +[t8] +[t9] diff --git a/scripts/fio/create/create_files_1000.fio b/scripts/fio/create/create_files_1000.fio new file mode 100644 index 00000000..dcd9e978 --- /dev/null +++ b/scripts/fio/create/create_files_1000.fio @@ -0,0 +1,18 @@ +[global] +create_on_open=1 +nrfiles=100 +ioengine=filecreate +fallocate=none +filesize=4k +openfiles=1 + +[t0] +[t1] +[t2] +[t3] +[t4] +[t5] +[t6] +[t7] +[t8] +[t9] diff --git a/scripts/fio/create/create_files_10000.fio b/scripts/fio/create/create_files_10000.fio new file mode 100644 index 00000000..d5ffea7d --- /dev/null +++ b/scripts/fio/create/create_files_10000.fio @@ -0,0 +1,18 @@ +[global] +create_on_open=1 +nrfiles=1000 +ioengine=filecreate +fallocate=none +filesize=4k +openfiles=1 + +[t0] +[t1] +[t2] +[t3] +[t4] +[t5] +[t6] +[t7] +[t8] +[t9] diff --git a/scripts/fio/create/create_files_100000.fio b/scripts/fio/create/create_files_100000.fio new file mode 100644 index 00000000..4ae273f1 --- /dev/null +++ b/scripts/fio/create/create_files_100000.fio @@ -0,0 +1,18 @@ +[global] +create_on_open=1 +nrfiles=10000 +ioengine=filecreate +fallocate=none +filesize=4k +openfiles=1 + +[t0] +[t1] +[t2] +[t3] +[t4] +[t5] +[t6] +[t7] +[t8] +[t9] diff --git a/scripts/fio/mix/mix_1r4w.fio b/scripts/fio/mix/mix_1r4w.fio new file mode 100644 index 00000000..c15025b5 --- /dev/null +++ b/scripts/fio/mix/mix_1r4w.fio @@ -0,0 +1,24 @@ +[global] +name=fs_bench +bs=256k +runtime=30s +time_based +group_reporting + +[sequential_read] +size=100G +rw=read +ioengine=sync +fallocate=none +numjobs=1 + +[sequential_write_four_threads] +new_group +numjobs=4 +size=100G +rw=write +ioengine=sync +fallocate=none +create_on_open=1 +fsync_on_close=1 +unlink=1 diff --git a/scripts/fio/mix/mix_2r2w.fio b/scripts/fio/mix/mix_2r2w.fio new file mode 100644 index 00000000..d670d7e1 --- /dev/null +++ b/scripts/fio/mix/mix_2r2w.fio @@ -0,0 +1,24 @@ +[global] +name=fs_bench +bs=256k +runtime=30s +time_based +group_reporting + +[sequential_read_two_threads] +size=100G +rw=read +ioengine=sync +fallocate=none +numjobs=2 + +[sequential_write_two_threads] +new_group +numjobs=2 +size=100G +rw=write +ioengine=sync +fallocate=none +create_on_open=1 +fsync_on_close=1 +unlink=1 diff --git a/scripts/fio/mix/mix_4r1w.fio b/scripts/fio/mix/mix_4r1w.fio new file mode 100644 index 00000000..ff706623 --- /dev/null +++ b/scripts/fio/mix/mix_4r1w.fio @@ -0,0 +1,24 @@ +[global] +name=fs_bench +bs=256k +runtime=30s +time_based +group_reporting + +[sequential_read_four_threads] +size=100G +rw=read +ioengine=sync +fallocate=none +numjobs=4 + +[sequential_write] +new_group +numjobs=1 +size=100G +rw=write +ioengine=sync +fallocate=none +create_on_open=1 +fsync_on_close=1 +unlink=1 diff --git a/scripts/fio/read/rand_read.fio b/scripts/fio/read/rand_read.fio new file mode 100644 index 00000000..72a4bfc8 --- /dev/null +++ b/scripts/fio/read/rand_read.fio @@ -0,0 +1,12 @@ +[global] +name=fs_bench +bs=256k +runtime=30s +time_based +group_reporting + +[random_read] +size=100G +rw=randread +ioengine=sync +fallocate=none diff --git a/scripts/fio/read/rand_read_4t.fio b/scripts/fio/read/rand_read_4t.fio new file mode 100644 index 00000000..70e38caa --- /dev/null +++ b/scripts/fio/read/rand_read_4t.fio @@ -0,0 +1,13 @@ +[global] +name=fs_bench +bs=256k +runtime=30s +time_based +group_reporting + +[random_read_four_threads] +size=100G +rw=randread +ioengine=sync +fallocate=none +numjobs=4 diff --git a/scripts/fio/read/rand_read_4t_direct.fio b/scripts/fio/read/rand_read_4t_direct.fio new file mode 100644 index 00000000..3ec15b48 --- /dev/null +++ b/scripts/fio/read/rand_read_4t_direct.fio @@ -0,0 +1,14 @@ +[global] +name=fs_bench +bs=256k +runtime=30s +time_based +group_reporting + +[random_read_four_threads_direct_io] +size=100G +rw=randread +ioengine=sync +fallocate=none +direct=1 +numjobs=4 diff --git a/scripts/fio/read/rand_read_4t_direct_small.fio b/scripts/fio/read/rand_read_4t_direct_small.fio new file mode 100644 index 00000000..b6eb1d9d --- /dev/null +++ b/scripts/fio/read/rand_read_4t_direct_small.fio @@ -0,0 +1,14 @@ +[global] +name=fs_bench +bs=256k +runtime=30s +time_based +group_reporting + +[random_read_four_threads_direct_io_small_file] +size=5m +rw=randread +ioengine=sync +fallocate=none +direct=1 +numjobs=4 diff --git a/scripts/fio/read/rand_read_4t_small.fio b/scripts/fio/read/rand_read_4t_small.fio new file mode 100644 index 00000000..9e7d75a2 --- /dev/null +++ b/scripts/fio/read/rand_read_4t_small.fio @@ -0,0 +1,13 @@ +[global] +name=fs_bench +bs=256k +runtime=30s +time_based +group_reporting + +[random_read_four_threads_small_file] +size=5m +rw=randread +ioengine=sync +fallocate=none +numjobs=4 diff --git a/scripts/fio/read/rand_read_direct.fio b/scripts/fio/read/rand_read_direct.fio new file mode 100644 index 00000000..2356822c --- /dev/null +++ b/scripts/fio/read/rand_read_direct.fio @@ -0,0 +1,13 @@ +[global] +name=fs_bench +bs=256k +runtime=30s +time_based +group_reporting + +[random_read_direct_io] +size=100G +rw=randread +ioengine=sync +fallocate=none +direct=1 diff --git a/scripts/fio/read/rand_read_direct_small.fio b/scripts/fio/read/rand_read_direct_small.fio new file mode 100644 index 00000000..b1162b2e --- /dev/null +++ b/scripts/fio/read/rand_read_direct_small.fio @@ -0,0 +1,13 @@ +[global] +name=fs_bench +bs=256k +runtime=30s +time_based +group_reporting + +[random_read_direct_io_small_file] +size=5m +rw=randread +ioengine=sync +fallocate=none +direct=1 diff --git a/scripts/fio/read/rand_read_small.fio b/scripts/fio/read/rand_read_small.fio new file mode 100644 index 00000000..8ca8c824 --- /dev/null +++ b/scripts/fio/read/rand_read_small.fio @@ -0,0 +1,12 @@ +[global] +name=fs_bench +bs=256k +runtime=30s +time_based +group_reporting + +[random_read_small_file] +size=5m +rw=randread +ioengine=sync +fallocate=none diff --git a/scripts/fio/read/seq_read.fio b/scripts/fio/read/seq_read.fio new file mode 100644 index 00000000..18bae644 --- /dev/null +++ b/scripts/fio/read/seq_read.fio @@ -0,0 +1,12 @@ +[global] +name=fs_bench +bs=256k +runtime=30s +time_based +group_reporting + +[sequential_read] +size=100G +rw=read +ioengine=sync +fallocate=none diff --git a/scripts/fio/read/seq_read_4t.fio b/scripts/fio/read/seq_read_4t.fio new file mode 100644 index 00000000..45933994 --- /dev/null +++ b/scripts/fio/read/seq_read_4t.fio @@ -0,0 +1,13 @@ +[global] +name=fs_bench +bs=256k +runtime=30s +time_based +group_reporting + +[sequential_read_four_threads] +size=100G +rw=read +ioengine=sync +fallocate=none +numjobs=4 diff --git a/scripts/fio/read/seq_read_4t_direct.fio b/scripts/fio/read/seq_read_4t_direct.fio new file mode 100644 index 00000000..66eccb2a --- /dev/null +++ b/scripts/fio/read/seq_read_4t_direct.fio @@ -0,0 +1,14 @@ +[global] +name=fs_bench +bs=256k +runtime=30s +time_based +group_reporting + +[sequential_read_four_threads_direct_io] +size=100G +rw=read +ioengine=sync +fallocate=none +direct=1 +numjobs=4 diff --git a/scripts/fio/read/seq_read_4t_direct_small.fio b/scripts/fio/read/seq_read_4t_direct_small.fio new file mode 100644 index 00000000..8d53a52a --- /dev/null +++ b/scripts/fio/read/seq_read_4t_direct_small.fio @@ -0,0 +1,14 @@ +[global] +name=fs_bench +bs=256k +runtime=30s +time_based +group_reporting + +[sequential_read_four_threads_direct_io_small_file] +size=5m +rw=read +ioengine=sync +fallocate=none +direct=1 +numjobs=4 diff --git a/scripts/fio/read/seq_read_4t_small.fio b/scripts/fio/read/seq_read_4t_small.fio new file mode 100644 index 00000000..52224593 --- /dev/null +++ b/scripts/fio/read/seq_read_4t_small.fio @@ -0,0 +1,13 @@ +[global] +name=fs_bench +bs=256k +runtime=30s +time_based +group_reporting + +[sequential_read_four_threads_small_file] +size=5m +rw=read +ioengine=sync +fallocate=none +numjobs=4 diff --git a/scripts/fio/read/seq_read_direct.fio b/scripts/fio/read/seq_read_direct.fio new file mode 100644 index 00000000..02e90915 --- /dev/null +++ b/scripts/fio/read/seq_read_direct.fio @@ -0,0 +1,13 @@ +[global] +name=fs_bench +bs=256k +runtime=30s +time_based +group_reporting + +[sequential_read_direct_io] +size=100G +rw=read +ioengine=sync +fallocate=none +direct=1 diff --git a/scripts/fio/read/seq_read_direct_small.fio b/scripts/fio/read/seq_read_direct_small.fio new file mode 100644 index 00000000..aef7cf0e --- /dev/null +++ b/scripts/fio/read/seq_read_direct_small.fio @@ -0,0 +1,13 @@ +[global] +name=fs_bench +bs=256k +runtime=30s +time_based +group_reporting + +[sequential_read_direct_io_small_file] +size=5m +rw=read +ioengine=sync +fallocate=none +direct=1 diff --git a/scripts/fio/read/seq_read_skip_17m.fio b/scripts/fio/read/seq_read_skip_17m.fio new file mode 100644 index 00000000..c73888ef --- /dev/null +++ b/scripts/fio/read/seq_read_skip_17m.fio @@ -0,0 +1,12 @@ +[global] +name=fs_bench +bs=256m +runtime=30s +time_based +group_reporting + +[seq_read_skip_17m] +size=100G +rw=read:17m +ioengine=sync +fallocate=none diff --git a/scripts/fio/read/seq_read_small.fio b/scripts/fio/read/seq_read_small.fio new file mode 100644 index 00000000..54a9a973 --- /dev/null +++ b/scripts/fio/read/seq_read_small.fio @@ -0,0 +1,12 @@ +[global] +name=fs_bench +bs=256k +runtime=30s +time_based +group_reporting + +[sequential_read_small_file] +size=5m +rw=read +ioengine=sync +fallocate=none diff --git a/scripts/fio/read_latency/ttfb.fio b/scripts/fio/read_latency/ttfb.fio new file mode 100644 index 00000000..07dbfe92 --- /dev/null +++ b/scripts/fio/read_latency/ttfb.fio @@ -0,0 +1,13 @@ +[global] +name=fs_bench +# large write block size not used as part of benchmark, but speeds up file layout +bs=1B,1M + +[time_to_first_byte_read] +size=1G +io_limit=1B +rw=read +ioengine=sync +fallocate=none +direct=1 +loops=10 diff --git a/scripts/fio/read_latency/ttfb_small.fio b/scripts/fio/read_latency/ttfb_small.fio new file mode 100644 index 00000000..f88be2ab --- /dev/null +++ b/scripts/fio/read_latency/ttfb_small.fio @@ -0,0 +1,11 @@ +[global] +name=fs_bench +bs=1B + +[time_to_first_byte_read_small_file] +size=1B +rw=read +ioengine=sync +fallocate=none +direct=1 +loops=10 diff --git a/scripts/fio/write/seq_write.fio b/scripts/fio/write/seq_write.fio new file mode 100644 index 00000000..871bb784 --- /dev/null +++ b/scripts/fio/write/seq_write.fio @@ -0,0 +1,15 @@ +[global] +name=fs_bench +bs=256k +runtime=30s +time_based +group_reporting + +[sequential_write] +size=100G +rw=write +ioengine=sync +fallocate=none +create_on_open=1 +fsync_on_close=1 +unlink=1 diff --git a/scripts/fio/write/seq_write_direct.fio b/scripts/fio/write/seq_write_direct.fio new file mode 100644 index 00000000..b56cc63c --- /dev/null +++ b/scripts/fio/write/seq_write_direct.fio @@ -0,0 +1,16 @@ +[global] +name=fs_bench +bs=256k +runtime=30s +time_based +group_reporting + +[sequential_write_direct_io] +size=100G +rw=write +ioengine=sync +direct=1 +fallocate=none +create_on_open=1 +fsync_on_close=1 +unlink=1 diff --git a/scripts/fio/write_latency/time_to_write_one_byte_file.fio b/scripts/fio/write_latency/time_to_write_one_byte_file.fio new file mode 100644 index 00000000..fa4b2a76 --- /dev/null +++ b/scripts/fio/write_latency/time_to_write_one_byte_file.fio @@ -0,0 +1,14 @@ +[global] +name=fs_bench +bs=1B + +[time_to_write_one_byte_file] +size=1B +rw=write +ioengine=sync +fallocate=none +create_on_open=1 +fsync_on_close=1 +unlink=1 +unlink_each_loop=1 +loops=10 diff --git a/scripts/fs_bench.sh b/scripts/fs_bench.sh new file mode 100755 index 00000000..b080bacd --- /dev/null +++ b/scripts/fs_bench.sh @@ -0,0 +1,510 @@ +#!/usr/bin/env bash +# Benchmark hf-mount throughput with fio — mirrors mountpoint-s3/scripts/fs_bench.sh. +# +# Required env vars: +# HF_TOKEN — HuggingFace API token +# +# Optional env vars: +# HF_ENDPOINT — defaults to https://huggingface.co +# HF_MOUNT_BIN — path to hf-mount-fuse (default: ./target/release/hf-mount-fuse) +# HF_MOUNT_BACKEND — "fuse" (default) or "nfs" +# HF_ADVANCED_WRITES — set to 1 to enable --advanced-writes +# HF_NO_DISK_CACHE — set to 1 to disable the on-disk xorb chunk cache +# HF_BENCH_BUCKET — reuse a pre-existing bucket (skips create/delete) +# HF_JOB_NAME_FILTER — only run jobs whose filename matches this substring +# HF_CATEGORIES — comma-separated list of categories to run +# (default: read,write,mix,read_latency,write_latency,create) +# iterations — fio iterations per job (default: 10) +# +# Usage: +# cargo build --release +# # FUSE with cache (default) +# ./scripts/fs_bench.sh +# # FUSE without cache +# HF_NO_DISK_CACHE=1 ./scripts/fs_bench.sh +# # FUSE with advanced writes +# HF_ADVANCED_WRITES=1 ./scripts/fs_bench.sh +# # NFS +# HF_MOUNT_BACKEND=nfs ./scripts/fs_bench.sh +# # Quick run (small files only) +# HF_JOB_NAME_FILTER=small ./scripts/fs_bench.sh +set -euo pipefail + +if ! command -v fio &>/dev/null; then + echo "fio must be installed: sudo apt-get install -y fio" >&2; exit 1 +fi +if ! command -v jq &>/dev/null; then + echo "jq must be installed: sudo apt-get install -y jq" >&2; exit 1 +fi +if [[ -z "${HF_TOKEN:-}" ]]; then + echo "Set HF_TOKEN to run this benchmark" >&2; exit 1 +fi + +HF_ENDPOINT="${HF_ENDPOINT:-https://huggingface.co}" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +BACKEND="${HF_MOUNT_BACKEND:-fuse}" +: "${iterations:=10}" + +# Build the mode label for results directory +MODE="${BACKEND}" +[[ "${HF_NO_DISK_CACHE:-0}" == "1" ]] && MODE="${MODE}_nocache" +[[ "${HF_ADVANCED_WRITES:-0}" == "1" ]] && MODE="${MODE}_advwr" + +# Resolve binary +if [[ "${BACKEND}" == "nfs" ]]; then + HF_MOUNT_BIN="${HF_MOUNT_BIN:-${PROJECT_DIR}/target/release/hf-mount-nfs}" +else + HF_MOUNT_BIN="${HF_MOUNT_BIN:-${PROJECT_DIR}/target/release/hf-mount-fuse}" +fi + +if [[ ! -x "${HF_MOUNT_BIN}" ]]; then + echo "Binary not found at ${HF_MOUNT_BIN}. Run: cargo build --release" >&2; exit 1 +fi + +# Categories to run +IFS=',' read -ra CATEGORIES <<< "${HF_CATEGORIES:-read,write,mix,read_latency,write_latency,create}" + +results_dir="${PROJECT_DIR}/results/${MODE}" +rm -rf "${results_dir}" +mkdir -p "${results_dir}" + +echo "=== Benchmark mode: ${MODE} ===" >&2 +echo "Binary: ${HF_MOUNT_BIN}" >&2 +echo "Categories: ${CATEGORIES[*]}" >&2 +[[ -n "${HF_JOB_NAME_FILTER:-}" ]] && echo "Job filter: ${HF_JOB_NAME_FILTER}" >&2 + +# ── Bucket lifecycle ────────────────────────────────────────────────────────── + +OWNS_BUCKET=false +if [[ -z "${HF_BENCH_BUCKET:-}" ]]; then + USERNAME=$(curl -sf -H "Authorization: Bearer ${HF_TOKEN}" \ + "${HF_ENDPOINT}/api/whoami-v2" | jq -r '.name') + HF_BENCH_BUCKET="${USERNAME}/hf-bench-$$" + curl -sf -X POST "${HF_ENDPOINT}/api/buckets/${HF_BENCH_BUCKET}" \ + -H "Authorization: Bearer ${HF_TOKEN}" -H "Content-Type: application/json" \ + -d '{}' >/dev/null + echo "Created bucket: ${HF_BENCH_BUCKET}" >&2 + OWNS_BUCKET=true +fi + +_CHILD_PID="" +_MOUNT_DIR="" + +cleanup() { + if [[ -n "${_MOUNT_DIR}" ]] && mountpoint -q "${_MOUNT_DIR}" 2>/dev/null; then + if [[ "${BACKEND}" == "nfs" ]]; then + sudo umount "${_MOUNT_DIR}" 2>/dev/null || true + else + fusermount -u "${_MOUNT_DIR}" 2>/dev/null || true + fi + fi + if [[ -n "${_CHILD_PID}" ]]; then + kill "${_CHILD_PID}" 2>/dev/null || true + wait "${_CHILD_PID}" 2>/dev/null || true + fi + if [[ "${OWNS_BUCKET}" == true ]]; then + curl -sf -X DELETE "${HF_ENDPOINT}/api/buckets/${HF_BENCH_BUCKET}" \ + -H "Authorization: Bearer ${HF_TOKEN}" >/dev/null 2>&1 || true + echo "Deleted bucket: ${HF_BENCH_BUCKET}" >&2 + fi +} +trap cleanup EXIT + +# ── Mount helpers ───────────────────────────────────────────────────────────── + +# Mount hf-mount-fuse; sets _CHILD_PID and _MOUNT_DIR. +do_mount() { + local cache_dir="$1" + shift + _MOUNT_DIR="$(mktemp -d /tmp/hf-bench-XXXXXXXXXX)" + mkdir -p "${cache_dir}" + + local mount_args=( + --hf-token "${HF_TOKEN}" + --hub-endpoint "${HF_ENDPOINT}" + --cache-dir "${cache_dir}" + --poll-interval-secs 0 + ) + [[ "${HF_NO_DISK_CACHE:-0}" == "1" ]] && mount_args+=(--no-disk-cache) + [[ "${HF_ADVANCED_WRITES:-0}" == "1" ]] && mount_args+=(--advanced-writes) + # When running as root (sudo), set file ownership to the calling user + # so fio and other tools can access mounted files without permission issues. + if [[ "$(id -u)" == "0" && -n "${SUDO_UID:-}" ]]; then + mount_args+=(--uid "${SUDO_UID}" --gid "${SUDO_GID:-${SUDO_UID}}") + fi + mount_args+=("$@") + mount_args+=(bucket "${HF_BENCH_BUCKET}" "${_MOUNT_DIR}") + + "${HF_MOUNT_BIN}" "${mount_args[@]}" &>/dev/null & + _CHILD_PID=$! + for i in $(seq 1 30); do + grep -q "${_MOUNT_DIR}" /proc/mounts 2>/dev/null && \ + { echo "Mount ready after $((i*500))ms" >&2; return 0; } + sleep 0.5 + done + echo "ERROR: mount not ready after 15s" >&2; exit 1 +} + +do_unmount() { + if [[ "${BACKEND}" == "nfs" ]]; then + sudo umount "${_MOUNT_DIR}" 2>/dev/null || true + else + fusermount -u "${_MOUNT_DIR}" 2>/dev/null || true + fi + for _ in $(seq 1 30); do + kill -0 "${_CHILD_PID}" 2>/dev/null || { _CHILD_PID=""; _MOUNT_DIR=""; return 0; } + sleep 1 + done + kill "${_CHILD_PID}" 2>/dev/null || true + wait "${_CHILD_PID}" 2>/dev/null || true + _CHILD_PID=""; _MOUNT_DIR="" +} + +should_run_job() { + [[ -z "${HF_JOB_NAME_FILTER:-}" ]] || [[ "$1" == *"${HF_JOB_NAME_FILTER}"* ]] +} + +# ── File setup (one-time) ───────────────────────────────────────────────────── +# Write all benchmark files through hf-mount's write path, flush to CAS, then unmount. +# This mirrors mountpoint-s3's "create_only" step, but batched for efficiency. + +if [[ "${OWNS_BUCKET}" == true ]]; then + echo "Laying out bench files (write-through to CAS)..." >&2 + do_mount "/tmp/hf-bench-setup-cache-$$" + for category in read write; do + jobs_dir="${SCRIPT_DIR}/fio/${category}" + [[ -d "${jobs_dir}" ]] || continue + for job_file in "${jobs_dir}"/*.fio; do + if ! { [[ -z "${HF_JOB_NAME_FILTER:-}" ]] || [[ "${job_file}" == *"${HF_JOB_NAME_FILTER}"* ]]; }; then + continue + fi + echo " Creating files for $(basename "${job_file}")" >&2 + fio --thread --directory="${_MOUNT_DIR}" --create_only=1 --eta=never "${job_file}" \ + &>/dev/null || true + done + done + echo "Unmounting to flush uploads to CAS (may take a while for large files)..." >&2 + do_unmount + echo "Files uploaded." >&2 +fi + +# ── fio benchmark runner ────────────────────────────────────────────────────── + +# Throughput aggregation (MiB/s) — same jq as mountpoint-s3, with null-safe rw check. +run_fio_job() { + local job_file="$1" mount_dir="$2" + local job_name + job_name="$(basename "${job_file}" .fio)" + + echo -n "Running job ${job_name} for ${iterations} iterations... " >&2 + for i in $(seq 1 "${iterations}"); do + echo -n "${i};" >&2 + set +e + timeout 900s fio \ + --thread \ + --output="${results_dir}/${job_name}_iter${i}.json" \ + --output-format=json \ + --directory="${mount_dir}" \ + --eta=never \ + "${job_file}" + local job_status=$? + set -e + if [[ ${job_status} -ne 0 ]]; then + echo "Job ${job_name} failed with exit code ${job_status}" >&2; exit 1 + fi + done + echo "done" >&2 + + jq -s '[ + [.[].jobs] | add | group_by(.jobname)[] | + { + name: .[0].jobname, + value: ((map( + if (.["job options"].rw // "write") | test("^(rand)?read") + then .read.bw + else .write.bw + end + ) | add) / (length * 1024)), + unit: "MiB/s" + } + ] | { + name: map(.name) | unique | join(","), + value: map(.value) | add, + unit: "MiB/s" + }' "${results_dir}/${job_name}_iter"*.json | tee "${results_dir}/${job_name}_parsed.json" +} + +# Latency aggregation (milliseconds). +run_fio_latency_job() { + local job_file="$1" mount_dir="$2" + local job_name + job_name="$(basename "${job_file}" .fio)" + + echo -n "Running latency job ${job_name}... " >&2 + set +e + timeout 900s fio \ + --thread \ + --output="${results_dir}/${job_name}_iter1.json" \ + --output-format=json \ + --directory="${mount_dir}" \ + --eta=never \ + "${job_file}" + local job_status=$? + set -e + if [[ ${job_status} -ne 0 ]]; then + echo "Job ${job_name} failed with exit code ${job_status}" >&2; exit 1 + fi + echo "done" >&2 + + jq -n 'inputs.jobs[] | + if (.["job options"].rw // "write") | test("^(rand)?read") + then {name: .jobname, value: (.read.lat_ns.mean / 1000000), unit: "milliseconds"} + else {name: .jobname, value: (.write.lat_ns.mean / 1000000), unit: "milliseconds"} + end + ' "${results_dir}/${job_name}_iter1.json" | tee "${results_dir}/${job_name}_parsed.json" +} + +# Create benchmark aggregation (files/s via IOPS). +run_fio_create_job() { + local job_file="$1" mount_dir="$2" + local job_name + job_name="$(basename "${job_file}" .fio)" + + echo -n "Running create job ${job_name} for ${iterations} iterations... " >&2 + for i in $(seq 1 "${iterations}"); do + echo -n "${i};" >&2 + set +e + timeout 900s fio \ + --thread \ + --output="${results_dir}/${job_name}_iter${i}.json" \ + --output-format=json \ + --directory="${mount_dir}" \ + --eta=never \ + "${job_file}" + local job_status=$? + set -e + if [[ ${job_status} -ne 0 ]]; then + echo "Job ${job_name} failed with exit code ${job_status}" >&2; exit 1 + fi + done + echo "done" >&2 + + # Sum IOPS across all threads, average across iterations + jq -s '{ + name: .[0].jobs[0].jobname, + value: ([.[].jobs | map(.write.iops) | add] | add / length), + unit: "files/s" + }' "${results_dir}/${job_name}_iter"*.json | tee "${results_dir}/${job_name}_parsed.json" +} + +# ── Category runners ────────────────────────────────────────────────────────── + +# Generate a temporary fio write job from a read job (for populating test files). +# Strips time_based/runtime so fio writes the full file, changes rw to write, +# and removes direct=1 (not useful for layout writes). +make_write_job() { + local read_job="$1" + local write_job + write_job=$(mktemp /tmp/fio-write-XXXX.fio) + # create_on_open=1: skip the layout step (create+ftruncate) and create at open time, + # so fio opens with O_CREAT|O_TRUNC which streaming write mode supports. + # Remove numjobs so populate writes a single copy of each file. + sed -e 's/rw=\(rand\)\?read/rw=write/' \ + -e '/^time_based/d' \ + -e '/^runtime=/d' \ + -e '/^direct=/d' \ + -e '/^numjobs=/d' \ + "${read_job}" > "${write_job}" + echo "create_on_open=1" >> "${write_job}" + echo "${write_job}" +} + +# Read category: write real data first, then benchmark reads on a fresh mount. +# This ensures reads hit CAS instead of reading sparse zeros. +run_read_category() { + local jobs_dir="${SCRIPT_DIR}/fio/read" + [[ -d "${jobs_dir}" ]] || return 0 + + # Collect jobs to run + local job_files=() + for job_file in "${jobs_dir}"/*.fio; do + should_run_job "${job_file}" || continue + job_files+=("${job_file}") + done + [[ ${#job_files[@]} -gt 0 ]] || return 0 + + # Phase 1: populate all test files on a single writable mount + # Skip when reusing a pre-existing bucket (files are already there). + if [[ -z "${HF_BENCH_BUCKET:-}" ]]; then + local populate_cache="/tmp/hf-bench-populate-$$" + do_mount "${populate_cache}" + for job_file in "${job_files[@]}"; do + echo "Populating data for $(basename "${job_file}")" >&2 + local write_job + write_job=$(make_write_job "${job_file}") + fio --thread --directory="${_MOUNT_DIR}" --eta=never "${write_job}" &>/dev/null + rm -f "${write_job}" + done + echo "Unmounting to commit writes..." >&2 + do_unmount + rm -rf "${_MOUNT_DIR}" 2>/dev/null || true + else + echo "Skipping populate (reusing existing bucket ${HF_BENCH_BUCKET})" >&2 + fi + + # Phase 2: benchmark each job on a fresh cold-cache mount + for job_file in "${job_files[@]}"; do + local job_name cache_dir + job_name="$(basename "${job_file}" .fio)" + cache_dir="/tmp/hf-bench-read-$$/${job_name}" + + do_mount "${cache_dir}" + run_fio_job "${job_file}" "${_MOUNT_DIR}" + do_unmount + rm -rf "${_MOUNT_DIR}" 2>/dev/null || true + done +} + +# Standard throughput categories (write, mix). +run_throughput_category() { + local category="$1" + local jobs_dir="${SCRIPT_DIR}/fio/${category}" + [[ -d "${jobs_dir}" ]] || return 0 + + for job_file in "${jobs_dir}"/*.fio; do + if ! { [[ -z "${HF_JOB_NAME_FILTER:-}" ]] || [[ "${job_file}" == *"${HF_JOB_NAME_FILTER}"* ]]; }; then + echo "Skipping $(basename "${job_file}")" >&2 + continue + fi + + local job_name cache_dir + job_name="$(basename "${job_file}" .fio)" + cache_dir="/tmp/hf-bench-cache-$$/${job_name}" + + # Mount read-write so fio can create and benchmark on the same mount. + local extra_args=() + [[ "${HF_NO_DISK_CACHE:-0}" == "1" ]] && extra_args+=(--no-disk-cache) + do_mount "${cache_dir}" "${extra_args[@]}" + + echo "Laying out files for ${job_file}" >&2 + fio --thread --directory="${_MOUNT_DIR}" --create_only=1 --eta=never "${job_file}" &>/dev/null + + run_fio_job "${job_file}" "${_MOUNT_DIR}" + do_unmount + rm -rf "${_MOUNT_DIR}" 2>/dev/null || true + done +} + +# Read latency: write real data, unmount, remount read-only, measure TTFB. +run_read_latency() { + local jobs_dir="${SCRIPT_DIR}/fio/read_latency" + [[ -d "${jobs_dir}" ]] || return 0 + + local setup_cache="/tmp/hf-bench-latency-setup-$$" + + # Populate all latency test files with real data on one mount + echo "Populating read_latency files..." >&2 + do_mount "${setup_cache}" + for job_file in "${jobs_dir}"/*.fio; do + should_run_job "${job_file}" || continue + local write_job + write_job=$(make_write_job "${job_file}") + fio --thread --directory="${_MOUNT_DIR}" --eta=never "${write_job}" &>/dev/null || true + rm -f "${write_job}" + done + echo "Unmounting to commit writes..." >&2 + do_unmount + + # Benchmark each job on a fresh read-only mount + for job_file in "${jobs_dir}"/*.fio; do + if ! should_run_job "${job_file}"; then + echo "Skipping $(basename "${job_file}")" >&2 + continue + fi + + local job_name cache_dir + job_name="$(basename "${job_file}" .fio)" + cache_dir="/tmp/hf-bench-latency-$$/${job_name}" + + do_mount "${cache_dir}" --read-only + + run_fio_latency_job "${job_file}" "${_MOUNT_DIR}" + + do_unmount + rm -rf "${_MOUNT_DIR}" 2>/dev/null || true + done +} + +# Write latency. +run_write_latency() { + local jobs_dir="${SCRIPT_DIR}/fio/write_latency" + [[ -d "${jobs_dir}" ]] || return 0 + + for job_file in "${jobs_dir}"/*.fio; do + if ! should_run_job "${job_file}"; then + echo "Skipping $(basename "${job_file}")" >&2 + continue + fi + + local job_name cache_dir + job_name="$(basename "${job_file}" .fio)" + cache_dir="/tmp/hf-bench-cache-$$/${job_name}" + + do_mount "${cache_dir}" + + run_fio_latency_job "${job_file}" "${_MOUNT_DIR}" + + do_unmount + rm -rf "${_MOUNT_DIR}" 2>/dev/null || true + done +} + +# Create benchmark (files/s). +run_create() { + local jobs_dir="${SCRIPT_DIR}/fio/create" + [[ -d "${jobs_dir}" ]] || return 0 + + for job_file in "${jobs_dir}"/*.fio; do + if ! should_run_job "${job_file}"; then + echo "Skipping $(basename "${job_file}")" >&2 + continue + fi + + local job_name cache_dir + job_name="$(basename "${job_file}" .fio)" + cache_dir="/tmp/hf-bench-cache-$$/${job_name}" + + do_mount "${cache_dir}" + + run_fio_create_job "${job_file}" "${_MOUNT_DIR}" + + do_unmount + rm -rf "${_MOUNT_DIR}" 2>/dev/null || true + done +} + +# ── Main ────────────────────────────────────────────────────────────────────── + +for cat in "${CATEGORIES[@]}"; do + echo "" >&2 + echo "=== Category: ${cat} ===" >&2 + case "${cat}" in + read) + run_read_category ;; + write|mix) + run_throughput_category "${cat}" ;; + read_latency) + run_read_latency ;; + write_latency) + run_write_latency ;; + create) + run_create ;; + *) + echo "Unknown category: ${cat}" >&2 ;; + esac +done + +echo "" >&2 +echo "=== Results (${MODE}) ===" >&2 +jq -n '[inputs]' "${results_dir}"/*_parsed.json | tee "${results_dir}/output.json" diff --git a/src/hub_api.rs b/src/hub_api.rs index 2a0f4151..7a74eaf2 100644 --- a/src/hub_api.rs +++ b/src/hub_api.rs @@ -278,8 +278,20 @@ impl HubApiClient { let (client, head_client) = make_clients(); let endpoint = endpoint.trim_end_matches('/').to_string(); + // Read token from file if no inline token was provided. + let file_token = if token.is_none() { + token_file + .as_ref() + .and_then(|p| std::fs::read_to_string(p).ok()) + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) + } else { + None + }; + let effective_token: Option<&str> = token.or(file_token.as_deref()); + let auth = |req: reqwest::RequestBuilder| -> reqwest::RequestBuilder { - match token { + match effective_token { Some(t) => req.bearer_auth(t), None => req, } diff --git a/src/setup.rs b/src/setup.rs index 140abccc..a9987057 100644 --- a/src/setup.rs +++ b/src/setup.rs @@ -103,7 +103,8 @@ pub struct MountOptions { /// Disable the on-disk xorb chunk cache. Every read fetches chunks from /// the CAS network (no local disk storage between reads). Useful for - /// benchmarking without cache effects. + /// benchmarking without cache effects, comparable to mountpoint-s3 + /// without --cache. #[arg(long, default_value_t = false)] pub no_disk_cache: bool, diff --git a/src/virtual_fs/tests.rs b/src/virtual_fs/tests.rs index 5f36cc54..37d9efee 100644 --- a/src/virtual_fs/tests.rs +++ b/src/virtual_fs/tests.rs @@ -969,27 +969,6 @@ fn setattr_simple_mode_noop() { }); } -/// setattr(size) on an empty file in simple mode is a silent noop. -#[test] -fn setattr_simple_mode_empty_file_noop() { - let hub = MockHub::new(); - let xet = MockXet::new(); - let (rt, vfs) = vfs_simple(&hub, &xet); - - rt.block_on(async { - let (attr, _fh) = vfs - .create(ROOT_INODE, "new.txt", 0o644, 1000, 1000, Some(1)) - .await - .unwrap(); - // ftruncate(fd, N) in simple mode succeeds but is a noop - let result = vfs.setattr(attr.ino, Some(100), None, None, None, None, None).await; - assert!(result.is_ok(), "ftruncate should succeed (noop): {:?}", result); - let inodes = vfs.inode_table.read().unwrap(); - let entry = inodes.get(attr.ino).unwrap(); - assert_eq!(entry.size, 0, "size should be unchanged (noop)"); - }); -} - /// setattr(size) on a directory returns EISDIR. #[test] fn setattr_directory_eisdir() { diff --git a/tests/bench.rs b/tests/bench.rs deleted file mode 100644 index 1bb8ffee..00000000 --- a/tests/bench.rs +++ /dev/null @@ -1,165 +0,0 @@ -mod common; - -const BENCH_SIZES: &[(usize, &str)] = &[ - (50 * 1024 * 1024, "50MB"), - (200 * 1024 * 1024, "200MB"), - (500 * 1024 * 1024, "500MB"), -]; - -#[tokio::test] -async fn test_bench() { - // Upload files of each size to the same bucket - let (token, bucket_id, hub) = match common::setup_bucket("bench").await { - Some(cfg) => cfg, - None => return, - }; - - let mut files: Vec<(String, Vec)> = Vec::new(); - for &(size, label) in BENCH_SIZES { - let filename = format!("bench_{}.bin", label); - let data = common::generate_pattern(size); - let write_config = common::build_write_config(&hub).await; - - let tmp_dir = std::env::temp_dir().join(format!("hf-mount-bench-setup-{}", label)); - std::fs::create_dir_all(&tmp_dir).ok(); - let staging_path = tmp_dir.join(&filename); - std::fs::write(&staging_path, &data).expect("write staging file"); - - let file_info = common::upload_file(write_config, &staging_path).await; - let xet_hash = file_info.hash().to_string(); - eprintln!("Uploaded {}: xet_hash={}", label, xet_hash); - - let mtime_ms = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap_or_default() - .as_millis() as u64; - - hub.batch_operations(&[hf_mount::hub_api::BatchOp::AddFile { - path: filename.clone(), - xet_hash, - mtime: mtime_ms, - content_type: None, - }]) - .await - .expect("batch add failed"); - - std::fs::remove_dir_all(&tmp_dir).ok(); - files.push((filename, data)); - } - - let pid = std::process::id(); - - // --- FUSE benchmark (read + write per size) --- - let fuse_mount = format!("/tmp/hf-bench-fuse-{}", pid); - let fuse_cache = format!("/tmp/hf-bench-fuse-cache-{}", pid); - - let fuse_result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - let child = common::mount_bucket(&bucket_id, &fuse_mount, &fuse_cache, &[]); - - let mut reads = Vec::new(); - for (filename, expected) in &files { - reads.push(common::bench::run_read_benchmarks(&fuse_mount, filename, expected)); - } - - let mut writes = Vec::new(); - for &(size, _) in BENCH_SIZES { - writes.push(common::bench::run_write_benchmark(&fuse_mount, size)); - } - - common::unmount(&fuse_mount, child, 60); - (reads, writes) - })); - - std::fs::remove_dir_all(&fuse_mount).ok(); - std::fs::remove_dir_all(&fuse_cache).ok(); - - // --- NFS benchmark (read only) --- - let nfs_mount = format!("/tmp/hf-bench-nfs-{}", pid); - let nfs_cache = format!("/tmp/hf-bench-nfs-cache-{}", pid); - - let nfs_result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - let child = common::mount_bucket_nfs(&bucket_id, &nfs_mount, &nfs_cache, &["--read-only"]); - let mut reads = Vec::new(); - for (filename, expected) in &files { - reads.push(common::bench::run_read_benchmarks(&nfs_mount, filename, expected)); - } - common::unmount_nfs(&nfs_mount, child, 5); - reads - })); - - std::fs::remove_dir_all(&nfs_mount).ok(); - std::fs::remove_dir_all(&nfs_cache).ok(); - - // Cleanup - common::delete_bucket(&common::endpoint(), &token, &bucket_id).await; - - // --- Extract results --- - let (fuse_reads, fuse_writes) = match fuse_result { - Ok((reads, writes)) => (reads, writes), - Err(e) => std::panic::resume_unwind(e), - }; - - let nfs_reads = match nfs_result { - Ok(reads) => reads, - Err(e) => std::panic::resume_unwind(e), - }; - - // --- Print tables per size --- - for (i, &(_, label)) in BENCH_SIZES.iter().enumerate() { - let fuse = fuse_reads[i].as_ref().ok(); - let nfs = nfs_reads[i].as_ref().ok(); - let write = fuse_writes[i].as_ref().ok(); - - eprintln!("\n============================================================"); - eprintln!(" Benchmark — {}", label); - eprintln!("------------------------------------------------------------"); - eprintln!(" {:30} {:>12} {:>12}", "Metric", "FUSE", "NFS"); - eprintln!(" {:-<30} {:-<12} {:-<12}", "", "", ""); - - if let (Some(f), Some(n)) = (fuse, nfs) { - eprintln!( - " {:30} {:>9.1} MB/s {:>9.1} MB/s", - "Sequential read", f.seq_read_mbps, n.seq_read_mbps - ); - eprintln!( - " {:30} {:>9.1} MB/s {:>9.1} MB/s", - "Sequential re-read", f.seq_reread_mbps, n.seq_reread_mbps - ); - eprintln!( - " {:30} {:>9.1} ms {:>9.1} ms", - "Range read (1MB@25MB)", f.range_read_ms, n.range_read_ms - ); - eprintln!( - " {:30} {:>9.1} ms {:>9.1} ms", - "Random reads (100x4KB avg)", f.random_avg_ms, n.random_avg_ms - ); - } else { - if fuse.is_none() { - eprintln!(" FUSE read: FAILED"); - } - if nfs.is_none() { - eprintln!(" NFS read: FAILED"); - } - } - - if let Some(w) = write { - eprintln!(" {:30} {:>9.1} MB/s", "Sequential write (FUSE)", w.write_mbps); - eprintln!(" {:30} {:>9.3} s", "Close latency (CAS+Hub)", w.close_secs); - eprintln!(" {:30} {:>9.1} MB/s", "Write end-to-end", w.total_mbps); - eprintln!(" {:30} {:>9.1} MB/s", "Dedup write", w.dedup_write_mbps); - eprintln!(" {:30} {:>9.3} s", "Dedup close latency", w.dedup_close_secs); - eprintln!(" {:30} {:>9.1} MB/s", "Dedup end-to-end", w.dedup_total_mbps); - } else { - eprintln!(" Write: FAILED"); - } - eprintln!("============================================================"); - } - eprintln!(); - - // Assertions - for (i, &(_, label)) in BENCH_SIZES.iter().enumerate() { - assert!(fuse_reads[i].is_ok(), "FUSE read benchmark failed for {}", label); - assert!(nfs_reads[i].is_ok(), "NFS read benchmark failed for {}", label); - assert!(fuse_writes[i].is_ok(), "Write benchmark failed for {}", label); - } -} diff --git a/tests/common/bench.rs b/tests/common/bench.rs deleted file mode 100644 index 21f903c1..00000000 --- a/tests/common/bench.rs +++ /dev/null @@ -1,213 +0,0 @@ -use std::io::{Read, Seek, SeekFrom, Write}; -use std::time::{Duration, Instant}; - -use hf_mount::xet::XetOps; - -type BenchError = Box; - -pub struct BenchResult { - pub seq_read_mbps: f64, - pub seq_reread_mbps: f64, - pub range_read_ms: f64, - pub random_avg_ms: f64, - pub neg_cache_first_ms: f64, - pub neg_cache_second_ms: f64, -} - -pub struct WriteBenchResult { - pub size_mb: f64, - pub write_mbps: f64, - pub close_secs: f64, - pub total_mbps: f64, - /// Second write of identical data (dedup should make close near-instant). - pub dedup_write_mbps: f64, - pub dedup_close_secs: f64, - pub dedup_total_mbps: f64, - /// Raw add_data() throughput without FUSE overhead. - pub raw_add_data_mbps: f64, -} - -/// Run the standard read benchmark suite on a mounted filesystem. -/// Returns structured results for comparison or printing. -pub fn run_read_benchmarks(mount_point: &str, test_filename: &str, expected: &[u8]) -> Result { - let file_path = format!("{}/{}", mount_point, test_filename); - let file_size = expected.len(); - let size_mb = file_size as f64 / (1024.0 * 1024.0); - - // 1. Sequential read (cold) - let seq_read_mbps; - { - let t = Instant::now(); - let data = std::fs::read(&file_path)?; - let elapsed = t.elapsed(); - seq_read_mbps = size_mb / elapsed.as_secs_f64(); - assert_eq!(data.len(), expected.len(), "size mismatch"); - assert!(data == expected, "content mismatch on sequential read"); - } - - // 2. Sequential re-read - let seq_reread_mbps; - { - let t = Instant::now(); - let data = std::fs::read(&file_path)?; - let elapsed = t.elapsed(); - seq_reread_mbps = size_mb / elapsed.as_secs_f64(); - assert_eq!(data.len(), expected.len()); - } - - // 3. Range read: 1MB at 25MB - let range_read_ms; - { - let offset = 25 * 1024 * 1024_usize; - let read_size = 1024 * 1024_usize; - let mut f = std::fs::File::open(&file_path)?; - let t = Instant::now(); - f.seek(SeekFrom::Start(offset as u64))?; - let mut buf = vec![0u8; read_size]; - f.read_exact(&mut buf)?; - range_read_ms = t.elapsed().as_secs_f64() * 1000.0; - assert!(super::verify_pattern(&buf, offset), "range read content mismatch"); - } - - // 4. Random reads: 100x 4KB - let random_avg_ms; - { - let read_size = 4096_usize; - let max_offset = file_size - read_size; - let mut rng_state: u64 = 42; - let mut total = Duration::ZERO; - for _ in 0..100 { - rng_state = rng_state.wrapping_mul(6364136223846793005).wrapping_add(1); - let offset = (rng_state % max_offset as u64) as usize; - - let mut f = std::fs::File::open(&file_path)?; - let t = Instant::now(); - f.seek(SeekFrom::Start(offset as u64))?; - let mut buf = vec![0u8; read_size]; - f.read_exact(&mut buf)?; - total += t.elapsed(); - - assert!( - super::verify_pattern(&buf, offset), - "random read mismatch at offset {}", - offset - ); - } - random_avg_ms = total.as_secs_f64() * 1000.0 / 100.0; - } - - // 5. Negative cache - let neg_cache_first_ms; - let neg_cache_second_ms; - { - let fake_paths: Vec = (0..20) - .map(|i| format!("{}/nonexistent_file_{}.txt", mount_point, i)) - .collect(); - - let t1 = Instant::now(); - for p in &fake_paths { - let _ = std::fs::metadata(p); - } - neg_cache_first_ms = t1.elapsed().as_secs_f64() * 1000.0; - - let t2 = Instant::now(); - for p in &fake_paths { - let _ = std::fs::metadata(p); - } - neg_cache_second_ms = t2.elapsed().as_secs_f64() * 1000.0; - } - - Ok(BenchResult { - seq_read_mbps, - seq_reread_mbps, - range_read_ms, - random_avg_ms, - neg_cache_first_ms, - neg_cache_second_ms, - }) -} - -/// Write benchmark: create a file of `size_bytes` via 1 MB chunks, measure -/// write throughput and close latency (close = CAS finalize + Hub commit). -pub fn run_write_benchmark(mount_point: &str, size_bytes: usize) -> Result { - let size_mb = size_bytes as f64 / (1024.0 * 1024.0); - let path = format!("{}/bench_write_{}.bin", mount_point, std::process::id()); - let chunk = super::generate_pattern(1024 * 1024); // 1 MB chunk - - // Write phase: measure wall time for all write() calls - let mut f = std::fs::File::create(&path)?; - let write_start = Instant::now(); - let mut remaining = size_bytes; - while remaining > 0 { - let n = remaining.min(chunk.len()); - f.write_all(&chunk[..n])?; - remaining -= n; - } - let write_elapsed = write_start.elapsed(); - - // Close phase: flush + drop triggers CAS finalize + Hub commit - let close_start = Instant::now(); - drop(f); - let close_elapsed = close_start.elapsed(); - - let total = write_elapsed + close_elapsed; - let write_mbps = size_mb / write_elapsed.as_secs_f64(); - let total_mbps = size_mb / total.as_secs_f64(); - - // Verify size - let meta = std::fs::metadata(&path)?; - assert_eq!(meta.len(), size_bytes as u64, "written file size mismatch"); - - // Second write: identical data → CAS dedup should make close near-instant - let path2 = format!("{}/bench_write_dedup_{}.bin", mount_point, std::process::id()); - let mut f2 = std::fs::File::create(&path2)?; - let write2_start = Instant::now(); - let mut remaining = size_bytes; - while remaining > 0 { - let n = remaining.min(chunk.len()); - f2.write_all(&chunk[..n])?; - remaining -= n; - } - let write2_elapsed = write2_start.elapsed(); - - let close2_start = Instant::now(); - drop(f2); - let close2_elapsed = close2_start.elapsed(); - - let total2 = write2_elapsed + close2_elapsed; - - Ok(WriteBenchResult { - size_mb, - write_mbps, - close_secs: close_elapsed.as_secs_f64(), - total_mbps, - dedup_write_mbps: size_mb / write2_elapsed.as_secs_f64(), - dedup_close_secs: close2_elapsed.as_secs_f64(), - dedup_total_mbps: size_mb / total2.as_secs_f64(), - raw_add_data_mbps: 0.0, // filled by run_raw_cleaner_benchmark - }) -} - -/// Measure raw SingleFileCleaner::add_data() throughput — no FUSE, no kernel. -/// This is the theoretical ceiling for streaming writes. -pub async fn run_raw_cleaner_benchmark(xet: &hf_mount::xet::XetSessions, size_bytes: usize) -> Result { - let size_mb = size_bytes as f64 / (1024.0 * 1024.0); - let chunk = super::generate_pattern(1024 * 1024); - - let mut writer = xet.create_streaming_writer().await.map_err(|e| e.to_string())?; - - let start = Instant::now(); - let mut remaining = size_bytes; - while remaining > 0 { - let n = remaining.min(chunk.len()); - writer.write(&chunk[..n]).await.map_err(|e| e.to_string())?; - remaining -= n; - } - let elapsed = start.elapsed(); - let mbps = size_mb / elapsed.as_secs_f64(); - - // Finish to clean up the session - let _ = writer.finish_boxed().await; - - Ok(mbps) -} diff --git a/tests/common/mod.rs b/tests/common/mod.rs index f11e4759..dfe923d5 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -1,6 +1,5 @@ #![allow(dead_code)] -pub mod bench; pub mod fs_tests; use std::path::Path; diff --git a/tests/fio_bench.rs b/tests/fio_bench.rs deleted file mode 100644 index 1fe504ab..00000000 --- a/tests/fio_bench.rs +++ /dev/null @@ -1,264 +0,0 @@ -mod common; - -use std::process::Command; -use std::time::{SystemTime, UNIX_EPOCH}; - -struct FioResult { - name: String, - bw_mbps: f64, - iops: f64, - lat_avg_us: f64, -} - -struct FioJob { - name: &'static str, - filename: &'static str, - rw: &'static str, - bs: &'static str, - time_based: bool, -} - -const FIO_JOBS: &[FioJob] = &[ - FioJob { - name: "seq-read-100M", - filename: "large_0.bin", - rw: "read", - bs: "128k", - time_based: false, - }, - FioJob { - name: "seq-reread-100M", - filename: "large_0.bin", - rw: "read", - bs: "128k", - time_based: false, - }, - FioJob { - name: "rand-read-4k-100M", - filename: "large_0.bin", - rw: "randread", - bs: "4k", - time_based: true, - }, - FioJob { - name: "seq-read-5x10M", - filename: "medium_0.bin:medium_1.bin:medium_2.bin:medium_3.bin:medium_4.bin", - rw: "read", - bs: "128k", - time_based: false, - }, - FioJob { - name: "rand-read-10x1M", - filename: "small_0.bin:small_1.bin:small_2.bin:small_3.bin:small_4.bin:small_5.bin:small_6.bin:small_7.bin:small_8.bin:small_9.bin", - rw: "randread", - bs: "4k", - time_based: true, - }, -]; - -fn run_fio_suite(mount_point: &str) -> Vec { - FIO_JOBS.iter().map(|job| run_fio(mount_point, job)).collect() -} - -fn run_fio(mount_point: &str, job: &FioJob) -> FioResult { - eprintln!(" fio: {}", job.name); - - let mut args = vec![ - "--name", - job.name, - "--directory", - mount_point, - "--readonly", - "--filename", - job.filename, - "--rw", - job.rw, - "--bs", - job.bs, - "--ioengine", - "sync", - "--output-format", - "json", - ]; - if job.time_based { - args.extend_from_slice(&["--runtime", "5", "--time_based"]); - } - - let output = Command::new("fio").args(&args).output().expect("Failed to run fio"); - - assert!( - output.status.success(), - "fio {} failed: {}", - job.name, - String::from_utf8_lossy(&output.stderr) - ); - - let json: serde_json::Value = serde_json::from_slice(&output.stdout).expect("fio JSON parse failed"); - - let read = &json["jobs"][0]["read"]; - let bw_bytes = read["bw"].as_f64().unwrap_or(0.0); // KiB/s - let iops = read["iops"].as_f64().unwrap_or(0.0); - let lat_avg_ns = read["lat_ns"]["mean"].as_f64().unwrap_or(0.0); - - FioResult { - name: job.name.to_string(), - bw_mbps: bw_bytes / 1024.0, // KiB/s -> MiB/s - iops, - lat_avg_us: lat_avg_ns / 1000.0, - } -} - -fn print_table(fuse_results: &[FioResult], nfs_results: &[FioResult]) { - eprintln!("\n============================================================"); - eprintln!(" fio Benchmark Results"); - eprintln!("------------------------------------------------------------"); - eprintln!( - " {:25} {:>10} {:>10} {:>10} {:>10}", - "Job", "FUSE MB/s", "NFS MB/s", "FUSE IOPS", "NFS IOPS" - ); - eprintln!( - " {:25} {:>10} {:>10} {:>10} {:>10}", - "-------------------------", "----------", "----------", "----------", "----------" - ); - - for (f, n) in fuse_results.iter().zip(nfs_results.iter()) { - let fuse_bw = format!("{:.1}", f.bw_mbps); - let nfs_bw = format!("{:.1}", n.bw_mbps); - - if f.name.contains("rand") { - // For random reads, IOPS and latency are more interesting - let fuse_iops = format!("{:.0}", f.iops); - let nfs_iops = format!("{:.0}", n.iops); - eprintln!( - " {:25} {:>10} {:>10} {:>10} {:>10}", - f.name, fuse_bw, nfs_bw, fuse_iops, nfs_iops - ); - } else { - eprintln!(" {:25} {:>10} {:>10} {:>10} {:>10}", f.name, fuse_bw, nfs_bw, "", ""); - } - } - - // Latency sub-table for random reads - let randoms: Vec<_> = fuse_results - .iter() - .zip(nfs_results.iter()) - .filter(|(f, _)| f.name.contains("rand")) - .collect(); - - if !randoms.is_empty() { - eprintln!(" {:25} {:>12} {:>12}", "Random Read Latency", "FUSE avg", "NFS avg"); - eprintln!( - " {:25} {:>12} {:>12}", - "-------------------------", "------------", "------------" - ); - for (f, n) in &randoms { - eprintln!(" {:25} {:>9.1} us {:>9.1} us", f.name, f.lat_avg_us, n.lat_avg_us); - } - } - - eprintln!("============================================================"); -} - -#[tokio::test] -async fn test_fio_compare() { - if Command::new("fio").arg("--version").output().is_err() { - eprintln!("Skipping: fio not installed"); - return; - } - - let (token, bucket_id, hub) = match common::setup_bucket("fio-cmp").await { - Some(cfg) => cfg, - None => return, - }; - - let write_config = common::build_write_config(&hub).await; - - let pid = std::process::id(); - let tmp_dir = std::env::temp_dir().join(format!("hf-fio-cmp-setup-{}", pid)); - std::fs::create_dir_all(&tmp_dir).ok(); - - let mut batch_ops = Vec::new(); - let mtime_ms = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_default() - .as_millis() as u64; - - let file_specs: Vec<(String, usize)> = (0..10) - .map(|i| (format!("small_{}.bin", i), 1024 * 1024)) - .chain((0..5).map(|i| (format!("medium_{}.bin", i), 10 * 1024 * 1024))) - .chain(std::iter::once(("large_0.bin".to_string(), 100 * 1024 * 1024))) - .collect(); - - let total_mb: usize = file_specs.iter().map(|(_, s)| s).sum::() / (1024 * 1024); - eprintln!("Uploading {} files ({} MB total)...", file_specs.len(), total_mb); - - for (filename, size) in &file_specs { - let data = common::generate_pattern(*size); - let staging_path = tmp_dir.join(filename); - std::fs::write(&staging_path, &data).expect("write staging file"); - - let file_info = common::upload_file(write_config.clone(), &staging_path).await; - let xet_hash = file_info.hash().to_string(); - eprintln!( - " Uploaded {} ({} MB) hash={}", - filename, - size / (1024 * 1024), - &xet_hash[..16] - ); - - batch_ops.push(hf_mount::hub_api::BatchOp::AddFile { - path: filename.clone(), - xet_hash, - mtime: mtime_ms, - content_type: None, - }); - } - - hub.batch_operations(&batch_ops).await.expect("batch add failed"); - eprintln!("All files committed to bucket"); - std::fs::remove_dir_all(&tmp_dir).ok(); - - // --- FUSE fio --- - let fuse_mount = format!("/tmp/hf-fio-cmp-fuse-{}", pid); - let fuse_cache = format!("/tmp/hf-fio-cmp-fuse-cache-{}", pid); - - eprintln!("\nRunning fio suite on FUSE..."); - let fuse_result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - let child = common::mount_bucket(&bucket_id, &fuse_mount, &fuse_cache, &["--read-only"]); - let results = run_fio_suite(&fuse_mount); - common::unmount(&fuse_mount, child, 5); - results - })); - - std::fs::remove_dir_all(&fuse_mount).ok(); - std::fs::remove_dir_all(&fuse_cache).ok(); - - // --- NFS fio --- - let nfs_mount = format!("/tmp/hf-fio-cmp-nfs-{}", pid); - let nfs_cache = format!("/tmp/hf-fio-cmp-nfs-cache-{}", pid); - - eprintln!("\nRunning fio suite on NFS..."); - let nfs_result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - let child = common::mount_bucket_nfs(&bucket_id, &nfs_mount, &nfs_cache, &["--read-only"]); - let results = run_fio_suite(&nfs_mount); - common::unmount_nfs(&nfs_mount, child, 5); - results - })); - - std::fs::remove_dir_all(&nfs_mount).ok(); - std::fs::remove_dir_all(&nfs_cache).ok(); - - // Cleanup - common::delete_bucket(&common::endpoint(), &token, &bucket_id).await; - - let fuse_results = match fuse_result { - Ok(r) => r, - Err(e) => std::panic::resume_unwind(e), - }; - let nfs_results = match nfs_result { - Ok(r) => r, - Err(e) => std::panic::resume_unwind(e), - }; - - print_table(&fuse_results, &nfs_results); -} diff --git a/tests/pjdfstest.rs b/tests/pjdfstest.rs index 856774fb..b815bdf3 100644 --- a/tests/pjdfstest.rs +++ b/tests/pjdfstest.rs @@ -1,5 +1,6 @@ mod common; +use std::fs::File; use std::process::Command; /// Expected results (established 2026-03-06). @@ -36,6 +37,13 @@ const PJDFSTEST_DIR: &str = "/tmp/pjdfstest"; const PJDFSTEST_REV: &str = "03eb25706d8dbf3611c3f820b45b7a5e09a36c06"; fn ensure_pjdfstest() -> bool { + // Acquire a file lock to prevent FUSE and NFS tests from racing on the build. + let lock_path = format!("{PJDFSTEST_DIR}.lock"); + let lock_file = File::create(&lock_path).expect("failed to create pjdfstest lock file"); + use std::os::unix::io::AsRawFd; + let fd = lock_file.as_raw_fd(); + unsafe { libc::flock(fd, libc::LOCK_EX) }; + let binary = format!("{}/pjdfstest", PJDFSTEST_DIR); // Verify both binary existence AND correct revision to avoid stale cache on self-hosted runners. if std::path::Path::new(&binary).exists() { @@ -51,9 +59,12 @@ fn ensure_pjdfstest() -> bool { eprintln!("Cached pjdfstest has wrong revision, rebuilding..."); } - // Remove stale directory (e.g. from interrupted previous run on self-hosted runner) + // Remove stale directory (e.g. from interrupted previous run on self-hosted runner). + // Use rm -rf as fallback since remove_dir_all can fail on root-owned files left by pjdfstest. + let _ = std::fs::remove_dir_all(PJDFSTEST_DIR); if std::path::Path::new(PJDFSTEST_DIR).exists() { - std::fs::remove_dir_all(PJDFSTEST_DIR).ok(); + eprintln!("Stale {PJDFSTEST_DIR} remains after remove_dir_all, trying rm -rf..."); + Command::new("rm").args(["-rf", PJDFSTEST_DIR]).status().ok(); } eprintln!("Building pjdfstest..."); @@ -248,39 +259,61 @@ fn print_results(results: &ProveResults) { } } -#[tokio::test] -async fn test_pjdfstest() { +fn preflight() -> bool { let is_ci = std::env::var("CI").is_ok(); if !ensure_pjdfstest() { if is_ci { panic!("pjdfstest build failed in CI -- setup error"); } eprintln!("Skipping: pjdfstest not available"); - return; + return false; } - - // prove requires perl if Command::new("prove").arg("--version").output().is_err() { if is_ci { panic!("prove (perl TAP harness) not found in CI"); } eprintln!("Skipping: prove (perl TAP harness) not installed"); + return false; + } + true +} + +fn assert_results(results: &ProveResults) { + let passed_tests = results.total_tests.saturating_sub(results.failed_tests); + assert!( + results.passed_files >= EXPECTED_FILES_PASS, + "Regression: {}/{} test files passed (expected at least {})", + results.passed_files, + results.total_files, + EXPECTED_FILES_PASS + ); + assert!( + passed_tests >= EXPECTED_TESTS_PASS, + "Regression: {}/{} tests passed (expected at least {})", + passed_tests, + results.total_tests, + EXPECTED_TESTS_PASS + ); +} + +#[tokio::test] +async fn test_pjdfstest_fuse() { + if !preflight() { return; } - let (token, bucket_id, _hub) = match common::setup_bucket("pjdfs").await { + let (token, bucket_id, _hub) = match common::setup_bucket("pjdfs-fuse").await { Some(cfg) => cfg, None => return, }; let pid = std::process::id(); - let mount_point = format!("/tmp/hf-pjdfs-{}", pid); - let cache_dir = format!("/tmp/hf-pjdfs-cache-{}", pid); + let mount_point = format!("/tmp/hf-pjdfs-fuse-{}", pid); + let cache_dir = format!("/tmp/hf-pjdfs-fuse-cache-{}", pid); let child = common::mount_bucket(&bucket_id, &mount_point, &cache_dir, &["--advanced-writes"]); let results = run_prove(&mount_point); - print_results(&results); common::unmount(&mount_point, child, 5); @@ -288,20 +321,33 @@ async fn test_pjdfstest() { std::fs::remove_dir_all(&mount_point).ok(); std::fs::remove_dir_all(&cache_dir).ok(); - // Exact regression assertions — all filtered tests must pass - let passed_tests = results.total_tests.saturating_sub(results.failed_tests); - assert!( - results.passed_files >= EXPECTED_FILES_PASS, - "Regression: {}/{} test files passed (expected at least {})", - results.passed_files, - results.total_files, - EXPECTED_FILES_PASS - ); - assert!( - passed_tests >= EXPECTED_TESTS_PASS, - "Regression: {}/{} tests passed (expected at least {})", - passed_tests, - results.total_tests, - EXPECTED_TESTS_PASS - ); + assert_results(&results); +} + +#[tokio::test] +async fn test_pjdfstest_nfs() { + if !preflight() { + return; + } + + let (token, bucket_id, _hub) = match common::setup_bucket("pjdfs-nfs").await { + Some(cfg) => cfg, + None => return, + }; + + let pid = std::process::id(); + let mount_point = format!("/tmp/hf-pjdfs-nfs-{}", pid); + let cache_dir = format!("/tmp/hf-pjdfs-nfs-cache-{}", pid); + + let child = common::mount_bucket_nfs(&bucket_id, &mount_point, &cache_dir, &["--advanced-writes"]); + + let results = run_prove(&mount_point); + print_results(&results); + + common::unmount_nfs(&mount_point, child, 5); + common::delete_bucket(&common::endpoint(), &token, &bucket_id).await; + std::fs::remove_dir_all(&mount_point).ok(); + std::fs::remove_dir_all(&cache_dir).ok(); + + assert_results(&results); }