Make AudioConverter Sendable #663
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Qwen3-ASR Smoke Test | |
| on: | |
| pull_request: | |
| branches: [main] | |
| workflow_dispatch: | |
| jobs: | |
| qwen3-asr-smoke-test: | |
| name: Qwen3-ASR Smoke Test (int8) | |
| runs-on: macos-15 | |
| permissions: | |
| contents: read | |
| pull-requests: write | |
| timeout-minutes: 45 | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - uses: swift-actions/setup-swift@v2 | |
| with: | |
| swift-version: "6.1" | |
| - name: Cache Dependencies | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| .build | |
| ~/Library/Application Support/FluidAudio/Models/qwen3-asr-0.6b-coreml | |
| ~/Library/Application Support/FluidAudio/Datasets/LibriSpeech | |
| ~/Library/Caches/Homebrew | |
| /usr/local/Cellar/ffmpeg | |
| /opt/homebrew/Cellar/ffmpeg | |
| key: ${{ runner.os }}-qwen3-asr-${{ hashFiles('Package.resolved', 'Sources/FluidAudio/Frameworks/**', 'Sources/FluidAudio/ModelRegistry.swift', 'Sources/FluidAudio/ModelNames.swift') }} | |
| - name: Install ffmpeg | |
| run: | | |
| brew install ffmpeg || echo "ffmpeg may already be installed" | |
| - name: Build | |
| run: swift build -c release | |
| - name: Run Smoke Test | |
| id: smoketest | |
| run: | | |
| BENCHMARK_START=$(date +%s) | |
| set -o pipefail | |
| echo "=========================================" | |
| echo "Qwen3-ASR int8 smoke test (1 file)" | |
| echo "=========================================" | |
| echo "" | |
| echo "NOTE: CoreML State API (GPU-resident KV cache) does not" | |
| echo "work correctly on virtualized CI VMs. WER/RTFx numbers" | |
| echo "will be wrong. This test only verifies the pipeline" | |
| echo "(download, load, transcribe) does not crash." | |
| echo "" | |
| if .build/release/fluidaudiocli qwen3-benchmark \ | |
| --variant int8 --max-files 1 \ | |
| --output qwen3_results_int8.json 2>&1; then | |
| echo "Smoke test PASSED - pipeline completed without crash" | |
| echo "SMOKE_STATUS=PASSED" >> $GITHUB_OUTPUT | |
| else | |
| EXIT_CODE=$? | |
| echo "Smoke test FAILED with exit code $EXIT_CODE" | |
| echo "SMOKE_STATUS=FAILED" >> $GITHUB_OUTPUT | |
| fi | |
| # Calculate execution time before validation (needed for PR comment) | |
| EXECUTION_TIME=$(( ($(date +%s) - BENCHMARK_START) / 60 ))m$(( ($(date +%s) - BENCHMARK_START) % 60 ))s | |
| echo "EXECUTION_TIME=$EXECUTION_TIME" >> $GITHUB_OUTPUT | |
| # Extract RTFx metrics if results file exists | |
| if [ -f qwen3_results_int8.json ]; then | |
| MEDIAN_RTFx=$(jq -r '.summary.medianRTFx // "N/A"' qwen3_results_int8.json 2>/dev/null) | |
| OVERALL_RTFx=$(jq -r '.summary.overallRTFx // "N/A"' qwen3_results_int8.json 2>/dev/null) | |
| [ "$MEDIAN_RTFx" != "null" ] && [ "$MEDIAN_RTFx" != "N/A" ] && MEDIAN_RTFx=$(printf "%.2f" "$MEDIAN_RTFx") || MEDIAN_RTFx="N/A" | |
| [ "$OVERALL_RTFx" != "null" ] && [ "$OVERALL_RTFx" != "N/A" ] && OVERALL_RTFx=$(printf "%.2f" "$OVERALL_RTFx") || OVERALL_RTFx="N/A" | |
| echo "MEDIAN_RTFx=$MEDIAN_RTFx" >> $GITHUB_OUTPUT | |
| echo "OVERALL_RTFx=$OVERALL_RTFx" >> $GITHUB_OUTPUT | |
| # Validate RTFx - fail if 0 or N/A (indicates benchmark failure) | |
| if [ "$MEDIAN_RTFx" = "N/A" ] || [ "$MEDIAN_RTFx" = "0.00" ] || [ "$MEDIAN_RTFx" = "0" ] || [ "$MEDIAN_RTFx" = "0.0" ] || \ | |
| [ "$OVERALL_RTFx" = "N/A" ] || [ "$OVERALL_RTFx" = "0.00" ] || [ "$OVERALL_RTFx" = "0" ] || [ "$OVERALL_RTFx" = "0.0" ]; then | |
| echo "❌ CRITICAL: RTFx is 0 or N/A - benchmark failed to produce valid results" | |
| echo "Median RTFx: $MEDIAN_RTFx" | |
| echo "Overall RTFx: $OVERALL_RTFx" | |
| exit 1 | |
| fi | |
| else | |
| echo "❌ CRITICAL: Results file not found - benchmark failed" | |
| echo "MEDIAN_RTFx=N/A" >> $GITHUB_OUTPUT | |
| echo "OVERALL_RTFx=N/A" >> $GITHUB_OUTPUT | |
| exit 1 | |
| fi | |
| - name: Comment PR | |
| if: always() && github.event_name == 'pull_request' | |
| continue-on-error: true | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const status = '${{ steps.smoketest.outputs.SMOKE_STATUS }}'; | |
| const emoji = status === 'PASSED' ? '✅' : '❌'; | |
| const medianRTFx = '${{ steps.smoketest.outputs.MEDIAN_RTFx }}'; | |
| const overallRTFx = '${{ steps.smoketest.outputs.OVERALL_RTFx }}'; | |
| const body = `## Qwen3-ASR int8 Smoke Test ${emoji} | |
| | Check | Result | | |
| |-------|--------| | |
| | Build | ✅ | | |
| | Model download | ${emoji} | | |
| | Model load | ${emoji} | | |
| | Transcription pipeline | ${emoji} | | |
| | Decoder size | 571 MB (vs 1.1 GB f32) | | |
| ### Performance Metrics | |
| | Metric | CI Value | Expected on Apple Silicon | | |
| |--------|----------|--------------------------| | |
| | Median RTFx | ${medianRTFx}x | ~2.5x | | |
| | Overall RTFx | ${overallRTFx}x | ~2.5x | | |
| <sub>Runtime: ${{ steps.smoketest.outputs.EXECUTION_TIME }}</sub> | |
| <sub>**Note:** CI VM lacks physical GPU — CoreML MLState (macOS 15) KV cache produces degraded results on virtualized runners. On Apple Silicon: ~1.3% WER / 2.5x RTFx.</sub> | |
| <!-- fluidaudio-benchmark-qwen3-asr -->`; | |
| const { data: comments } = await github.rest.issues.listComments({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: context.issue.number, | |
| }); | |
| const existing = comments.find(c => | |
| c.body.includes('<!-- fluidaudio-benchmark-qwen3-asr -->') | |
| ); | |
| if (existing) { | |
| await github.rest.issues.updateComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| comment_id: existing.id, | |
| body: body | |
| }); | |
| } else { | |
| await github.rest.issues.createComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: context.issue.number, | |
| body: body | |
| }); | |
| } | |
| - name: Upload Results | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: qwen3-asr-results | |
| path: qwen3_results_*.json |