Skip to content

Commit e989c6c

Browse files
lukekimlukekim
authored
Add CI workflow with sccache integration test (#2)
* Update CI for spiceio rename and sccache integration test - Trigger on pull_request only (not push) - Rename binary references spio → spiceio - Add full lint pipeline: fmt, check, clippy, rustdoc - Add sccache integration test step with UNAS SMB credentials - Hardcode us-west-1 region, spiceio bucket, ai_platform_dev share - Upload release artifact as spiceio * Address CI PR review comments - Remove workflow_dispatch trigger (PR-only) - Add comment explaining self-hosted runner requirement - Use repository variables with defaults for SMB config - Skip integration test on fork PRs (secret unavailable) - Assert cache hits > 0 and write errors == 0 in test script * Add release workflow triggered on GitHub release creation Builds an optimized release binary on the self-hosted macOS runner, packages it as a tarball with SHA-256 checksum, and uploads both as release assets. * Add setup action for installing spiceio in other workflows Composite action that downloads a released spiceio binary, adds it to PATH, and starts the S3-to-SMB proxy in the background. Usage from another repo: - uses: spiceai/spiceio/.github/actions/setup@trunk with: smb-server: 192.168.3.148 smb-user: runner smb-pass: ${{ secrets.UNAS_SMB_PASS }} smb-share: ai_platform_dev Outputs the endpoint URL and PID for use in subsequent steps. * Address PR #2 review comments (round 2) - Fix grep|awk pipelines in test script to not fail under set -e - Add --clobber to gh release upload for safe re-runs - Skip CI on fork PRs to protect self-hosted runner from untrusted code * Trigger CI on PR sync (push to PR branch) * Address PR #2 review comments (round 3) - Make token input required (expression defaults don't evaluate in action metadata) - Use RUNNER_OS/RUNNER_ARCH to match release asset naming convention - Add cleanup step (if: always) to kill backgrounded spiceio process * Skip setup if spiceio is already running on the bind address * Use release-downloader action instead of gh CLI script * Use gh CLI instead of third-party action for release download * Address PR #2 review comments (round 4) - Zero sccache stats before warm build so assertion is warm-only - Verify SHA-256 checksum after downloading release asset - Remove uname fallbacks (RUNNER_OS/RUNNER_ARCH always set in Actions) - Fix CI: use env.SPICEIO_SMB_PASS instead of secrets context in if condition * Fix formatting (cargo fmt) * Fix CI failures and simplify setup action to use SMB URL syntax - Fix rustdoc: escape brackets in der_wrap doc comment - Fix integration test skip: check secret via env step, not secrets context - Simplify setup action: accept smb://user:pass@server/share URL instead of separate inputs for each field - smb-pass input overrides URL password (for secrets) * Fix CI: install AWS CLI, harden setup action - Install awscli via brew if missing on self-hosted runner - Fix SMB URL parsing for smb://user@server/share (no password in URL) - Check spiceio server header for skip detection, not just any HTTP response
1 parent 8796333 commit e989c6c

7 files changed

Lines changed: 333 additions & 3 deletions

File tree

‎.github/actions/setup/action.yml‎

Lines changed: 174 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,174 @@
1+
name: Setup spiceio
2+
description: Install a released version of spiceio and start the S3-to-SMB proxy
3+
4+
inputs:
5+
version:
6+
description: Release tag to install (e.g. "v0.1.0"). Use "latest" for the most recent release.
7+
required: false
8+
default: latest
9+
smb-url:
10+
description: >
11+
SMB connection URL: smb://user:pass@server/share or smb://user:pass@server:port/share.
12+
Password can also be provided separately via smb-pass.
13+
required: true
14+
smb-pass:
15+
description: SMB password (overrides password in smb-url if both provided)
16+
required: false
17+
default: ""
18+
bucket:
19+
description: Virtual S3 bucket name
20+
required: false
21+
default: spiceio
22+
region:
23+
description: AWS region to advertise
24+
required: false
25+
default: us-east-1
26+
bind:
27+
description: Listen address for the S3 endpoint
28+
required: false
29+
default: "127.0.0.1:8333"
30+
token:
31+
description: GitHub token for downloading release assets from private repos
32+
required: true
33+
34+
outputs:
35+
endpoint:
36+
description: The S3-compatible endpoint URL
37+
value: ${{ steps.start.outputs.endpoint }}
38+
pid:
39+
description: PID of the spiceio background process (empty if skipped)
40+
value: ${{ steps.start.outputs.pid }}
41+
42+
runs:
43+
using: composite
44+
steps:
45+
- name: Download and install spiceio
46+
id: install
47+
shell: bash
48+
env:
49+
GH_TOKEN: ${{ inputs.token }}
50+
run: |
51+
set -euo pipefail
52+
53+
REPO="spiceai/spiceio"
54+
INSTALL_DIR="${RUNNER_TEMP}/spiceio-bin"
55+
ASSET="spiceio-${RUNNER_OS}-${RUNNER_ARCH}.tar.gz"
56+
VERSION="${{ inputs.version }}"
57+
58+
mkdir -p "$INSTALL_DIR"
59+
60+
if [[ "$VERSION" == "latest" ]]; then
61+
gh release download --repo "$REPO" --pattern "$ASSET" --pattern "${ASSET}.sha256" --dir "$INSTALL_DIR"
62+
else
63+
gh release download "$VERSION" --repo "$REPO" --pattern "$ASSET" --pattern "${ASSET}.sha256" --dir "$INSTALL_DIR"
64+
fi
65+
66+
# Verify integrity
67+
cd "$INSTALL_DIR"
68+
shasum -a 256 -c "${ASSET}.sha256"
69+
70+
tar xzf "$ASSET" -C "$INSTALL_DIR"
71+
chmod +x "$INSTALL_DIR/spiceio"
72+
echo "$INSTALL_DIR" >> "$GITHUB_PATH"
73+
echo "pid_file=${RUNNER_TEMP}/spiceio.pid" >> "$GITHUB_OUTPUT"
74+
75+
- name: Start spiceio
76+
id: start
77+
shell: bash
78+
env:
79+
SMB_URL: ${{ inputs.smb-url }}
80+
SMB_PASS_OVERRIDE: ${{ inputs.smb-pass }}
81+
SPICEIO_BUCKET: ${{ inputs.bucket }}
82+
SPICEIO_REGION: ${{ inputs.region }}
83+
SPICEIO_BIND: ${{ inputs.bind }}
84+
run: |
85+
set -euo pipefail
86+
87+
ENDPOINT="http://${SPICEIO_BIND}"
88+
PID_FILE="${{ steps.install.outputs.pid_file }}"
89+
90+
# Skip if spiceio is already listening on the requested address
91+
# Verify it's spiceio by checking for the "spiceio" server header
92+
if curl -sf -I "$ENDPOINT/" 2>/dev/null | grep -qi "server: spiceio"; then
93+
echo "spiceio already running at $ENDPOINT — skipping start"
94+
echo "endpoint=$ENDPOINT" >> "$GITHUB_OUTPUT"
95+
echo "pid=" >> "$GITHUB_OUTPUT"
96+
echo "skipped=true" >> "$GITHUB_OUTPUT"
97+
exit 0
98+
fi
99+
100+
# Parse smb://user:pass@server:port/share
101+
# Strips the smb:// prefix, then splits on :, @, /
102+
URL="${SMB_URL#smb://}"
103+
104+
USERINFO="${URL%%@*}"
105+
HOSTPATH="${URL#*@}"
106+
107+
SPICEIO_SMB_USER="${USERINFO%%:*}"
108+
# Extract password from URL only if user:pass@ format is used
109+
if [[ "$USERINFO" == *:* ]]; then
110+
URL_PASS="${USERINFO#*:}"
111+
else
112+
URL_PASS=""
113+
fi
114+
# smb-pass input takes priority, then URL password
115+
if [[ -n "$SMB_PASS_OVERRIDE" ]]; then
116+
SPICEIO_SMB_PASS="$SMB_PASS_OVERRIDE"
117+
elif [[ -n "$URL_PASS" ]]; then
118+
SPICEIO_SMB_PASS="$URL_PASS"
119+
else
120+
echo "::error::No SMB password provided (use smb-pass input or include in smb-url)"
121+
exit 1
122+
fi
123+
124+
HOSTPORT="${HOSTPATH%%/*}"
125+
SPICEIO_SMB_SHARE="${HOSTPATH#*/}"
126+
127+
if [[ "$HOSTPORT" == *:* ]]; then
128+
SPICEIO_SMB_SERVER="${HOSTPORT%%:*}"
129+
SPICEIO_SMB_PORT="${HOSTPORT#*:}"
130+
else
131+
SPICEIO_SMB_SERVER="$HOSTPORT"
132+
SPICEIO_SMB_PORT="445"
133+
fi
134+
135+
export SPICEIO_SMB_SERVER SPICEIO_SMB_PORT SPICEIO_SMB_USER SPICEIO_SMB_PASS SPICEIO_SMB_SHARE
136+
export SPICEIO_BUCKET SPICEIO_REGION SPICEIO_BIND
137+
138+
spiceio &
139+
PID=$!
140+
echo "$PID" > "$PID_FILE"
141+
echo "pid=$PID" >> "$GITHUB_OUTPUT"
142+
echo "endpoint=$ENDPOINT" >> "$GITHUB_OUTPUT"
143+
echo "skipped=false" >> "$GITHUB_OUTPUT"
144+
145+
# Wait for readiness
146+
echo "Waiting for spiceio on ${SPICEIO_BIND}..."
147+
for i in $(seq 1 30); do
148+
if curl -sf -o /dev/null "$ENDPOINT/" 2>/dev/null; then
149+
echo "spiceio ready at $ENDPOINT (PID $PID)"
150+
exit 0
151+
fi
152+
if ! kill -0 "$PID" 2>/dev/null; then
153+
echo "::error::spiceio exited unexpectedly"
154+
exit 1
155+
fi
156+
sleep 1
157+
done
158+
echo "::error::spiceio failed to start within 30s"
159+
exit 1
160+
161+
- name: Register cleanup
162+
if: always() && steps.start.outputs.skipped != 'true'
163+
shell: bash
164+
run: |
165+
PID_FILE="${{ steps.install.outputs.pid_file }}"
166+
if [[ -f "$PID_FILE" ]]; then
167+
PID=$(cat "$PID_FILE")
168+
if kill -0 "$PID" 2>/dev/null; then
169+
echo "Stopping spiceio (PID $PID)"
170+
kill "$PID" 2>/dev/null || true
171+
wait "$PID" 2>/dev/null || true
172+
fi
173+
rm -f "$PID_FILE"
174+
fi

‎.github/workflows/ci.yml‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
name: CI
2+
3+
on:
4+
pull_request:
5+
types: [opened, synchronize, reopened]
6+
7+
concurrency:
8+
group: ${{ github.workflow }}-${{ github.ref }}
9+
cancel-in-progress: true
10+
11+
env:
12+
CARGO_TERM_COLOR: always
13+
CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse
14+
CARGO_NET_GIT_FETCH_WITH_CLI: true
15+
16+
jobs:
17+
ci:
18+
name: Format, lint, build, and test
19+
# Self-hosted macOS runner required for CommonCrypto FFI and NAS access.
20+
# Skip fork PRs to prevent untrusted code execution on self-hosted runners.
21+
if: github.event.pull_request.head.repo.full_name == github.repository
22+
runs-on: spiceai-macos
23+
permissions:
24+
contents: read
25+
26+
steps:
27+
- name: Check out repository
28+
uses: actions/checkout@v4
29+
30+
- name: Set up Rust toolchain
31+
uses: dtolnay/rust-toolchain@stable
32+
with:
33+
components: rustfmt,clippy
34+
35+
- name: Cache Rust build artifacts
36+
uses: Swatinem/rust-cache@v2
37+
38+
- name: Check formatting
39+
run: cargo fmt --all --check
40+
41+
- name: Run cargo check
42+
run: cargo check --locked --all-targets --all-features
43+
44+
- name: Run clippy
45+
run: cargo clippy --locked --all-targets --all-features -- -D warnings -D clippy::all -D clippy::cargo -A clippy::cargo-common-metadata
46+
47+
- name: Check rustdoc
48+
run: RUSTDOCFLAGS="-D warnings" cargo doc --locked --workspace --no-deps --document-private-items
49+
50+
- name: Build debug binary
51+
run: cargo build --locked
52+
53+
- name: Run unit tests
54+
run: cargo test --locked
55+
56+
- name: Check SMB credentials
57+
run: |
58+
if [[ -n "$SPICEIO_SMB_PASS" ]]; then
59+
echo "HAS_SMB_PASS=true" >> "$GITHUB_ENV"
60+
fi
61+
env:
62+
SPICEIO_SMB_PASS: ${{ secrets.UNAS_SMB_PASS }}
63+
64+
- name: Install AWS CLI
65+
if: ${{ env.HAS_SMB_PASS == 'true' }}
66+
run: |
67+
if ! command -v aws &>/dev/null; then
68+
brew install awscli
69+
fi
70+
71+
- name: Run sccache integration test
72+
# Skipped when UNAS_SMB_PASS secret is not configured (e.g. fork PRs)
73+
if: ${{ env.HAS_SMB_PASS == 'true' }}
74+
env:
75+
SPICEIO_SMB_SERVER: ${{ vars.SPICEIO_SMB_SERVER || '192.168.3.148' }}
76+
SPICEIO_SMB_USER: ${{ vars.SPICEIO_SMB_USER || 'runner' }}
77+
SPICEIO_SMB_PASS: ${{ secrets.UNAS_SMB_PASS }}
78+
SPICEIO_SMB_SHARE: ${{ vars.SPICEIO_SMB_SHARE || 'ai_platform_dev' }}
79+
SPICEIO_BUCKET: ${{ vars.SPICEIO_BUCKET || 'spiceio' }}
80+
SPICEIO_REGION: ${{ vars.SPICEIO_REGION || 'us-west-1' }}
81+
run: ./scripts/test-sccache.sh
82+
83+
- name: Build release artifact
84+
run: cargo build --release --locked --bin spiceio
85+
86+
- name: Upload release artifact
87+
uses: actions/upload-artifact@v4
88+
with:
89+
name: spiceio-${{ runner.os }}-${{ runner.arch }}
90+
path: target/release/spiceio
91+
if-no-files-found: error

‎.github/workflows/release.yml‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
name: Release
2+
3+
on:
4+
release:
5+
types: [created]
6+
7+
concurrency:
8+
group: ${{ github.workflow }}-${{ github.ref }}
9+
cancel-in-progress: true
10+
11+
env:
12+
CARGO_TERM_COLOR: always
13+
CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse
14+
CARGO_NET_GIT_FETCH_WITH_CLI: true
15+
16+
jobs:
17+
build:
18+
name: Build release binary
19+
# Self-hosted macOS runner required for CommonCrypto FFI
20+
runs-on: spiceai-macos
21+
permissions:
22+
contents: write
23+
24+
steps:
25+
- name: Check out repository
26+
uses: actions/checkout@v4
27+
28+
- name: Set up Rust toolchain
29+
uses: dtolnay/rust-toolchain@stable
30+
31+
- name: Cache Rust build artifacts
32+
uses: Swatinem/rust-cache@v2
33+
34+
- name: Build release binary
35+
run: cargo build --release --locked --bin spiceio
36+
37+
- name: Package binary
38+
run: |
39+
cd target/release
40+
tar czf spiceio-${{ runner.os }}-${{ runner.arch }}.tar.gz spiceio
41+
shasum -a 256 spiceio-${{ runner.os }}-${{ runner.arch }}.tar.gz > spiceio-${{ runner.os }}-${{ runner.arch }}.tar.gz.sha256
42+
43+
- name: Upload release assets
44+
env:
45+
GH_TOKEN: ${{ github.token }}
46+
run: |
47+
gh release upload "${{ github.event.release.tag_name }}" --clobber \
48+
target/release/spiceio-${{ runner.os }}-${{ runner.arch }}.tar.gz \
49+
target/release/spiceio-${{ runner.os }}-${{ runner.arch }}.tar.gz.sha256

‎scripts/test-sccache.sh‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -259,6 +259,7 @@ CARGO_TARGET_DIR="$TEST_TARGET_DIR" cargo build 2>&1
259259
echo ""
260260
echo "[test] === warm build (should hit cache) ==="
261261
rm -rf "$TEST_TARGET_DIR"
262+
sccache --zero-stats 2>/dev/null || true
262263
CARGO_TARGET_DIR="$TEST_TARGET_DIR" cargo build 2>&1
263264

264265
echo ""
@@ -267,3 +268,17 @@ echo "[test] sccache stats:"
267268
echo "======================================="
268269
sccache --show-stats
269270
echo "======================================="
271+
272+
# ── Verify cache hits ───────────────────────────────────────────────────────
273+
274+
STATS=$(sccache --show-stats 2>&1)
275+
CACHE_HITS=$(echo "$STATS" | grep -m1 "^Cache hits" | awk '{print $NF}' || echo "0")
276+
WRITE_ERRORS=$(echo "$STATS" | grep -m1 "Cache write errors" | awk '{print $NF}' || echo "0")
277+
278+
echo ""
279+
if [[ "${CACHE_HITS:-0}" -gt 0 && "${WRITE_ERRORS:-0}" -eq 0 ]]; then
280+
echo "[test] PASS: warm build got $CACHE_HITS cache hits, 0 write errors"
281+
else
282+
echo "[test] FAIL: expected cache hits > 0 (got ${CACHE_HITS:-0}) and write errors == 0 (got ${WRITE_ERRORS:-0})"
283+
exit 1
284+
fi

‎src/s3/router.rs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -978,7 +978,9 @@ async fn handle_complete_multipart_upload(
978978
return error_response(
979979
StatusCode::BAD_REQUEST,
980980
"InvalidPart",
981-
&format!("One or more of the specified parts could not be found. Part number: {pn}"),
981+
&format!(
982+
"One or more of the specified parts could not be found. Part number: {pn}"
983+
),
982984
);
983985
}
984986
}

‎src/smb/auth.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -346,7 +346,7 @@ pub fn wrap_spnego_auth(ntlmssp: &[u8]) -> Vec<u8> {
346346
der_wrap(0xa1, &seq)
347347
}
348348

349-
/// Wrap data in a DER TLV: [tag][length][data].
349+
/// Wrap data in a DER TLV: `[tag][length][data]`.
350350
fn der_wrap(tag: u8, data: &[u8]) -> Vec<u8> {
351351
let mut buf = Vec::with_capacity(1 + 4 + data.len());
352352
buf.push(tag);

‎src/smb/ops.rs‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -558,4 +558,3 @@ fn guess_content_type(key: &str) -> String {
558558
}
559559
.into()
560560
}
561-

0 commit comments

Comments
 (0)