feat: Add propagating of traceparent #12849
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmarking | |
| on: | |
| push: | |
| branches: | |
| - main | |
| pull_request: | |
| # Concurrency configuration: | |
| # - We use workflow-specific concurrency groups to prevent multiple benchmark runs on the same code, | |
| # as benchmarks are extremely resource-intensive and require dedicated device time on SauceLabs. | |
| # - For pull requests, we cancel in-progress runs when new commits are pushed to avoid wasting | |
| # expensive external testing resources and provide timely performance feedback. | |
| # - For main branch pushes, we never cancel benchmarks to ensure we have complete performance | |
| # baselines for every main branch commit, which are critical for performance regression detection. | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: ${{ github.event_name == 'pull_request' }} | |
| jobs: | |
| files-changed: | |
| name: Detect File Changes | |
| runs-on: ubuntu-latest | |
| outputs: | |
| run_benchmarking_for_prs: ${{ steps.changes.outputs.run_benchmarking_for_prs }} | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Get changed files | |
| id: changes | |
| uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 | |
| with: | |
| token: ${{ github.token }} | |
| filters: .github/file-filters.yml | |
| build-benchmark-test-target: | |
| name: Build App and Test Runner | |
| # Run the job only for PRs with related changes or non-PR events. | |
| if: github.event_name != 'pull_request' || needs.files-changed.outputs.run_benchmarking_for_prs == 'true' | |
| needs: files-changed | |
| runs-on: macos-14 | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - run: ./scripts/ci-select-xcode.sh 15.4 | |
| - uses: ruby/setup-ruby@v1 | |
| with: | |
| bundler-cache: true | |
| - name: Install old xcodegen | |
| # Install xcodegen 2.43.0 for since newer versions don't work with Xcode older than Xcode 16 | |
| # We need to set HOMEBREW_DEVELOPER=1 to allow installing from a local formula | |
| run: | | |
| HOMEBREW_DEVELOPER=1 brew install --formula ./scripts/xcodegen.rb | |
| brew pin xcodegen | |
| - run: make init-ci-build | |
| - run: make xcode-ci | |
| - name: Install SentryCli | |
| run: brew install getsentry/tools/sentry-cli | |
| - name: Cache iOS-Swift App and dSYM build products | |
| id: ios-swift-cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| DerivedData/Build/Products/Debug-iphoneos/iOS-Swift.app.dSYM | |
| DerivedData/Build/Products/Debug-iphoneos/iOS-Swift.app | |
| key: ios-swift-for-ui-testing-cache-key-${{ hashFiles('Samples/iOS-Swift/**') }}-${{ hashFiles('Sources/Sentry/**') }} | |
| - name: Cache iOS-Swift UI Test Runner App build product | |
| id: ios-swift-benchmark-runner-cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| DerivedData/Build/Products/Debug-iphoneos/iOS-Benchmarking-Runner.app | |
| key: ios-swift-for-ui-testing-cache-key-${{ hashFiles('Samples/iOS-Benchmarking/**') }} | |
| - run: bundle exec fastlane build_ios_swift_for_tests | |
| env: | |
| APP_STORE_CONNECT_KEY_ID: ${{ secrets.APP_STORE_CONNECT_KEY_ID }} | |
| APP_STORE_CONNECT_ISSUER_ID: ${{ secrets.APP_STORE_CONNECT_ISSUER_ID }} | |
| APP_STORE_CONNECT_KEY: ${{ secrets.APP_STORE_CONNECT_KEY }} | |
| FASTLANE_KEYCHAIN_PASSWORD: ${{ secrets.FASTLANE_KEYCHAIN_PASSWORD }} | |
| MATCH_GIT_PRIVATE_KEY: ${{ secrets.MATCH_GIT_PRIVATE_KEY }} | |
| MATCH_PASSWORD: ${{ secrets.MATCH_PASSWORD }} | |
| MATCH_USERNAME: ${{ secrets.MATCH_USERNAME }} | |
| - run: bundle exec fastlane build_ios_benchmark_test | |
| env: | |
| APP_STORE_CONNECT_KEY_ID: ${{ secrets.APP_STORE_CONNECT_KEY_ID }} | |
| APP_STORE_CONNECT_ISSUER_ID: ${{ secrets.APP_STORE_CONNECT_ISSUER_ID }} | |
| APP_STORE_CONNECT_KEY: ${{ secrets.APP_STORE_CONNECT_KEY }} | |
| FASTLANE_KEYCHAIN_PASSWORD: ${{ secrets.FASTLANE_KEYCHAIN_PASSWORD }} | |
| MATCH_GIT_PRIVATE_KEY: ${{ secrets.MATCH_GIT_PRIVATE_KEY }} | |
| MATCH_PASSWORD: ${{ secrets.MATCH_PASSWORD }} | |
| MATCH_USERNAME: ${{ secrets.MATCH_USERNAME }} | |
| - name: Upload dSYMs | |
| run: | | |
| sentry-cli --auth-token ${{ secrets.SENTRY_AUTH_TOKEN }} upload-dif --org sentry-sdks --project sentry-cocoa DerivedData/Build/Products/Debug-iphoneos/iOS-Swift.app.dSYM | |
| - name: Archiving DerivedData | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: DerivedData-Xcode | |
| path: | | |
| **/Debug-iphoneos/iOS-Swift.app | |
| **/Debug-iphoneos/iOS-Benchmarking-Runner.app | |
| - name: Run CI Diagnostics | |
| if: failure() | |
| run: ./scripts/ci-diagnostics.sh | |
| run-ui-tests-with-sauce: | |
| name: Run Benchmarks ${{matrix.suite}} | |
| # Run the job only for PRs with related changes or non-PR events. | |
| if: github.event_name != 'pull_request' || needs.files-changed.outputs.run_benchmarking_for_prs == 'true' | |
| runs-on: ubuntu-latest | |
| needs: | |
| [ | |
| files-changed, | |
| build-benchmark-test-target, | |
| ] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| suite: ["High-end device", "Mid-range device", "Low-end device"] | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - uses: actions/download-artifact@v5 | |
| with: | |
| name: DerivedData-Xcode | |
| - run: npm install -g [email protected] | |
| - name: Run Benchmarks in SauceLab | |
| id: run-benchmarks-in-sauce-lab | |
| env: | |
| SAUCE_USERNAME: ${{ secrets.SAUCE_USERNAME }} | |
| SAUCE_ACCESS_KEY: ${{ secrets.SAUCE_ACCESS_KEY }} | |
| # Note: We are not setting continue-on-error here, because we want the step to be marked as failed. | |
| run: | | |
| set -o pipefail && saucectl run \ | |
| --select-suite "${{matrix.suite}}" \ | |
| --config .sauce/benchmarking-config.yml \ | |
| --tags benchmark \ | |
| --verbose \ | |
| 2>&1 | tee output.log | |
| - name: Recovery - Extract Test ID from output | |
| id: should-retry-test | |
| # Note: We need to use always() here, because the previous run step might be marked as failed. | |
| if: ${{ always() && steps.run-benchmarks-in-sauce-lab.outcome == 'failure' }} | |
| uses: actions/github-script@v8 | |
| env: | |
| SAUCE_USERNAME: ${{ secrets.SAUCE_USERNAME }} | |
| SAUCE_ACCESS_KEY: ${{ secrets.SAUCE_ACCESS_KEY }} | |
| with: | |
| # Retry Logic: This step only runs when the benchmark test has failed. | |
| # We determine if the failure was due to SauceLabs infrastructure issues | |
| # (which can be fixed by retrying) or legitimate test failures (which cannot). | |
| # SauceLabs internal errors frequently cause CI failures that can be resolved | |
| # by re-running the test. See scripts/saucelabs-helpers.js for detailed logic. | |
| script: | | |
| const fs = require('fs'); | |
| const { execSync } = require('child_process'); | |
| console.log("Extracting test ID from output log"); | |
| const outputLog = fs.readFileSync('output.log', 'utf8'); | |
| // Lookup for the test ID in the output log | |
| // Note: The CLI output might change over time, so this might need to be updated. | |
| const match = outputLog.match(/https:\/\/app\.saucelabs\.com\/tests\/([^\s]+)/); | |
| const testId = match?.[1] ?? ''; | |
| if (!testId) { | |
| core.warning("No SauceLabs test ID found in CLI output, it might have changed, retrying..."); | |
| core.setOutput('RETRY_TEST', 'true'); | |
| return; | |
| } | |
| try { | |
| console.log(`Checking if the test exists in SauceLabs: ${testId}`); | |
| execSync(`saucectl jobs get ${testId}`, { | |
| env: process.env, | |
| stdio: 'inherit' | |
| }); | |
| console.log("Test exists but failed, not retrying."); | |
| core.setFailed('Test exists but failed'); | |
| } catch (error) { | |
| console.log("Failed to get job, retrying..."); | |
| core.setOutput('RETRY_TEST', 'true'); | |
| } | |
| - name: Run Benchmarks in SauceLab - Retry 1 | |
| id: run-benchmarks-in-sauce-lab-retry-1 | |
| # Note: We need to use always() here, because the previous run step might be marked as failed. | |
| if: ${{ always() && steps.should-retry-test.outputs.RETRY_TEST == 'true' }} | |
| env: | |
| SAUCE_USERNAME: ${{ secrets.SAUCE_USERNAME }} | |
| SAUCE_ACCESS_KEY: ${{ secrets.SAUCE_ACCESS_KEY }} | |
| run: | | |
| echo "::warning SauceLabs benchmark tests need to be retried" | |
| saucectl run \ | |
| --select-suite "${{matrix.suite}}" \ | |
| --config .sauce/benchmarking-config.yml \ | |
| --tags benchmark \ | |
| --verbose | |
| - name: Run CI Diagnostics | |
| if: failure() | |
| run: ./scripts/ci-diagnostics.sh | |
| benchmarking-required-check: | |
| needs: | |
| [ | |
| files-changed, | |
| build-benchmark-test-target, | |
| run-ui-tests-with-sauce, | |
| ] | |
| name: Benchmarking | |
| # This is necessary since a failed/skipped dependent job would cause this job to be skipped | |
| if: always() | |
| runs-on: ubuntu-latest | |
| steps: | |
| # If any jobs we depend on fails gets cancelled or times out, this job will fail. | |
| # Skipped jobs are not considered failures. | |
| - name: Check for failures | |
| if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') | |
| run: | | |
| echo "One of the benchmark jobs has failed." && exit 1 |