Skip to content

perf: Optimize engine builder, task visitor, and untracked file discovery #8955

perf: Optimize engine builder, task visitor, and untracked file discovery

perf: Optimize engine builder, task visitor, and untracked file discovery #8955

Workflow file for this run

name: Test
on:
pull_request:
push:
branches: [main]
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
permissions:
actions: write
contents: read
pull-requests: write
jobs:
find-changes:
name: Find path changes
runs-on: ubuntu-latest
timeout-minutes: 30
outputs:
is-release-pr: ${{ steps.check-release.outputs.is-release-pr }}
docs: ${{ steps.filter.outputs.docs }}
rest: ${{ steps.filter.outputs.rest }}
rust: ${{ steps.filter.outputs.rust }}
native-lib: ${{ steps.filter.outputs.native-lib }}
steps:
# Detect automated release PRs which only contain version bumps.
# These PRs are created by the release workflow after code has already
# been tested on main. Skipping tests on version-only changes saves CI time.
- name: Checkout
uses: actions/checkout@v4
- name: Check if automated release PR
id: check-release
uses: ./.github/actions/check-release-pr
- name: Check path changes
if: steps.check-release.outputs.is-release-pr != 'true'
id: filter
run: |
# Determine the base and head commits to compare
if [ "${{ github.event_name }}" == "pull_request" ]; then
# For pull requests, compare the base branch to the current HEAD
git fetch origin ${{ github.base_ref }}
BASE_COMMIT="origin/${{ github.base_ref }}"
HEAD_COMMIT="HEAD"
else
# For pushes, use the before and after SHAs
BASE_COMMIT="${{ github.event.before }}"
HEAD_COMMIT="${{ github.event.after }}"
fi
echo "Comparing changes between $BASE_COMMIT and $HEAD_COMMIT"
# Function to check if files in given paths have changed
check_path_changes() {
local name=$1
shift
local paths=("$@")
# Create a command that checks all paths
local cmd="git diff --name-only $BASE_COMMIT $HEAD_COMMIT -- "
for path in "${paths[@]}"; do
cmd+="\"$path\" "
done
# Run the command and check if there are any results
if eval "$cmd" | grep -q .; then
echo "$name=true" >> $GITHUB_OUTPUT
echo "Changes detected in $name paths"
else
echo "$name=false" >> $GITHUB_OUTPUT
echo "No changes in $name paths"
fi
}
# Function to make path checking more readable
check_paths() {
local name=$1
local path_string=$2
# Convert the comma-separated string to an array
IFS=',' read -ra path_array <<< "$path_string"
# Call the check_path_changes function with the name and paths
check_path_changes "$name" "${path_array[@]}"
}
# Check each path pattern with a more readable syntax
echo "Checking path patterns..."
check_paths "docs" "docs/"
check_paths "native-lib" "packages/turbo-repository/,crates/"
# Handle the "rest" pattern - files that are NOT in examples/ or docs/
CHANGED_FILES=$(git diff --name-only $BASE_COMMIT $HEAD_COMMIT)
# Filter to only include files that do NOT start with examples/ or docs/
FILES_NOT_IN_EXAMPLES_OR_DOCS=$(echo "$CHANGED_FILES" | grep -v -E "^(examples/|docs/)" || true)
if [ -n "$FILES_NOT_IN_EXAMPLES_OR_DOCS" ]; then
echo "rest=true" >> $GITHUB_OUTPUT
echo "Changes detected outside examples/ and docs/ directories"
else
echo "rest=false" >> $GITHUB_OUTPUT
echo "No changes outside examples/ and docs/ directories"
fi
# Detect if Rust/core code changed (requires Rust tests + integration tests)
# This excludes JS-only changes like packages/*, lockfile, etc.
RUST_PATTERNS="^(crates/|cli/|Cargo\.|rust-toolchain|\.cargo/|turborepo-tests/)"
RUST_CHANGES=$(echo "$CHANGED_FILES" | grep -E "$RUST_PATTERNS" || true)
if [ -n "$RUST_CHANGES" ]; then
echo "rust=true" >> $GITHUB_OUTPUT
echo "Rust/core changes detected:"
echo "$RUST_CHANGES"
else
echo "rust=false" >> $GITHUB_OUTPUT
echo "No Rust/core changes detected (JS-only change)"
fi
generate-integration-test-matrix:
name: Generate integration test matrix
needs:
- find-changes
if: ${{ needs.find-changes.outputs.is-release-pr != 'true' && needs.find-changes.outputs.rust == 'true' }}
runs-on: ubuntu-latest
timeout-minutes: 30
outputs:
test-paths: ${{ steps.generate-matrix.outputs.test-paths }}
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Generate test path matrix
id: generate-matrix
run: |
TEST_PATHS=$(command ls -1A turborepo-tests/integration/tests | jq -R -s -c 'split("\n") | map(select(length > 0))')
echo "test-paths=$TEST_PATHS" >> $GITHUB_OUTPUT
echo "Generated test paths: $TEST_PATHS"
# Check for .t files directly in turborepo-tests/integration/tests (not in subdirectories)
if find turborepo-tests/integration/tests -maxdepth 1 -name "*.t" -type f | grep -q .; then
echo "::error::Found .t files directly in turborepo-tests/integration/tests/ which would be excluded from the test matrix"
echo "::error::Test files must be placed in subdirectories under turborepo-tests/integration/tests/"
find turborepo-tests/integration/tests -maxdepth 1 -name "*.t" -type f
exit 1
fi
integration:
name: Integration (${{ matrix.os.runner }}, ${{ matrix.test-path }})
needs:
- find-changes
- generate-integration-test-matrix
runs-on: ${{ matrix.os.runner }}
timeout-minutes: 30
if: ${{ needs.find-changes.outputs.is-release-pr != 'true' && needs.find-changes.outputs.rust == 'true' && needs.generate-integration-test-matrix.result == 'success' }}
env:
TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
TURBO_TEAM: ${{ vars.TURBO_TEAM }}
TURBO_CACHE: remote:rw
SCCACHE_BUCKET: turborepo-sccache
SCCACHE_REGION: us-east-2
RUSTC_WRAPPER: ${{ !github.event.pull_request.head.repo.fork && 'sccache' || '' }}
CARGO_INCREMENTAL: 0
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
SCCACHE_IDLE_TIMEOUT: 0
SCCACHE_REQUEST_TIMEOUT: 30
SCCACHE_ERROR_LOG: error
strategy:
fail-fast: false
matrix:
os:
- runner: ubuntu-latest
- runner: macos-latest
- runner: windows-latest
test-path: ${{ fromJson(needs.generate-integration-test-matrix.outputs.test-paths) }}
steps:
# On Windows, set autocrlf to input so that when the repo is cloned down
# the fixtures retain their line endings and don't get updated to CRLF.
# We want this because this repo also contains the fixtures for our test cases
# and these fixtures have files that need stable file hashes. If we let git update
# the line endings on checkout, the file hashes will change.
# https://www.git-scm.com/book/en/v2/Customizing-Git-Git-Configuration#_core_autocrlf
- name: set crlf
if: matrix.os.runner == 'windows-latest'
shell: bash
run: git config --global core.autocrlf input
- uses: actions/checkout@v4
- name: Setup Environment
uses: ./.github/actions/setup-environment
with:
github-token: "${{ secrets.GITHUB_TOKEN }}"
node-version: "18.20.2" # TODO: Update integration tests with changed log output in Node.js 22
- name: Install Global Turbo
uses: ./.github/actions/install-global-turbo
- name: Install Bun
uses: oven-sh/setup-bun@4bc047ad259df6fc24a6c9b0f9a0cb08cf17fbe5 # v2.0.1
- name: Setup Graphviz
uses: ts-graphviz/setup-graphviz@v2
with:
macos-skip-brew-update: "true"
env:
HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK: true
- name: Cache Prysk
id: cache-prysk
uses: actions/cache@v4
with:
path: cli/.cram_env
key: prysk-venv-${{ matrix.os.runner }}
- name: Run sccache-cache
uses: mozilla-actions/sccache-action@v0.0.6
- name: Install cargo-nextest
uses: taiki-e/install-action@44c6d64aa62cd779e873306675c7a58e86d6d532
with:
tool: nextest
- name: Integration Tests
timeout-minutes: 45
run: |
if [ -z "${RUSTC_WRAPPER}" ]; then
unset RUSTC_WRAPPER
fi
turbo run test --filter=turborepo-tests-integration --color --env-mode=strict -- "tests/${{ matrix.test-path }}"
shell: bash
turbo_types_check:
name: "@turbo/types codegen check"
needs:
- find-changes
if: ${{ needs.find-changes.outputs.is-release-pr != 'true' && needs.find-changes.outputs.rust == 'true' }}
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Setup Environment
uses: ./.github/actions/setup-environment
with:
github-token: "${{ secrets.GITHUB_TOKEN }}"
- name: Install Global Turbo
uses: ./.github/actions/install-global-turbo
- name: Generate schemas from Rust
run: |
cargo run -p turborepo-schema-gen -- schema -o packages/turbo-types/schemas/schema.json
cargo run -p turborepo-schema-gen -- typescript -o packages/turbo-types/src/types/config-v2.ts
pnpm exec oxfmt packages/turbo-types/schemas/schema.json packages/turbo-types/src/types/config-v2.ts
- name: Check for uncommitted changes
run: |
if ! git diff --exit-code; then
echo "::error::Generated schema files are out of sync with Rust types"
echo "::error::Please run 'pnpm generate-schema' in packages/turbo-types and commit the changes"
git diff
exit 1
fi
rust_test:
strategy:
fail-fast: false
matrix:
os:
- name: macos
runner: macos-latest
- name: windows
runner: windows-latest
partition: [1, 2]
runs-on: ${{ matrix.os.runner }}
timeout-minutes: 30
needs:
- find-changes
if: ${{ needs.find-changes.outputs.is-release-pr != 'true' && needs.find-changes.outputs.rust == 'true' }}
name: Rust testing on ${{ matrix.os.name }} (partition ${{ matrix.partition }}/2)
env:
SCCACHE_BUCKET: turborepo-sccache
SCCACHE_REGION: us-east-2
RUSTC_WRAPPER: ${{ !github.event.pull_request.head.repo.fork && 'sccache' || '' }}
CARGO_INCREMENTAL: 0
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
SCCACHE_IDLE_TIMEOUT: 0
SCCACHE_REQUEST_TIMEOUT: 30
SCCACHE_ERROR_LOG: error
steps:
- name: Set git to use LF line endings
run: |
git config --global core.autocrlf false
git config --global core.eol lf
if: matrix.os.name == 'windows'
- name: Checkout
uses: actions/checkout@v4
- name: Setup Environment
uses: ./.github/actions/setup-environment
with:
github-token: "${{ secrets.GITHUB_TOKEN }}"
node: "false"
- name: Run sccache-cache
uses: mozilla-actions/sccache-action@v0.0.6
- name: Install cargo-nextest
uses: taiki-e/install-action@44c6d64aa62cd779e873306675c7a58e86d6d532
with:
tool: nextest
- name: Build tests
timeout-minutes: 15
# We explicitly unset RUSTC_WRAPPER if it is an empty string as causes build issues
run: |
if [ -z "${RUSTC_WRAPPER}" ]; then
unset RUSTC_WRAPPER
fi
if [ "$RUNNER_OS" == "Windows" ]; then
cargo nextest run --workspace --exclude turborepo-napi --no-run
else
cargo nextest run --workspace --no-run
fi
shell: bash
- name: Run tests
timeout-minutes: 30
run: |
if [ "$RUNNER_OS" == "Windows" ]; then
cargo nextest run --workspace --exclude turborepo-napi --partition hash:${{ matrix.partition }}/2
else
cargo nextest run --workspace --partition hash:${{ matrix.partition }}/2
fi
shell: bash
rust_test_ubuntu:
strategy:
fail-fast: false
matrix:
partition: [1, 2]
runs-on: ubuntu-latest
timeout-minutes: 45
needs:
- find-changes
if: ${{ needs.find-changes.outputs.is-release-pr != 'true' && needs.find-changes.outputs.rust == 'true' }}
name: Rust testing on ubuntu (partition ${{ matrix.partition }}/2)
env:
SCCACHE_BUCKET: turborepo-sccache
SCCACHE_REGION: us-east-2
RUSTC_WRAPPER: ${{ !github.event.pull_request.head.repo.fork && 'sccache' || '' }}
CARGO_INCREMENTAL: 0
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
SCCACHE_IDLE_TIMEOUT: 0
SCCACHE_REQUEST_TIMEOUT: 30
SCCACHE_ERROR_LOG: error
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Setup Environment
uses: ./.github/actions/setup-environment
with:
github-token: "${{ secrets.GITHUB_TOKEN }}"
node: "false"
- name: Run sccache-cache
uses: mozilla-actions/sccache-action@v0.0.6
- name: Install cargo-nextest
uses: taiki-e/install-action@44c6d64aa62cd779e873306675c7a58e86d6d532
with:
tool: nextest
- name: Install cargo-llvm-cov
uses: taiki-e/install-action@v2
with:
tool: cargo-llvm-cov
- name: Build tests
timeout-minutes: 15
# We explicitly unset RUSTC_WRAPPER if it is an empty string as causes build issues
run: |
if [ -z "${RUSTC_WRAPPER}" ]; then
unset RUSTC_WRAPPER
fi
cargo nextest run --workspace --no-run
shell: bash
- name: Run tests with coverage
timeout-minutes: 30
# We explicitly unset RUSTC_WRAPPER if it is an empty string as causes build issues
run: |
if [ -z "${RUSTC_WRAPPER}" ]; then
unset RUSTC_WRAPPER
fi
cargo llvm-cov nextest \
--workspace \
--exclude turborepo-napi \
--partition hash:${{ matrix.partition }}/2 \
--lcov \
--output-path coverage-${{ matrix.partition }}.lcov
shell: bash
- name: Upload coverage artifact
uses: actions/upload-artifact@v4
with:
name: coverage-partition-${{ matrix.partition }}
path: coverage-${{ matrix.partition }}.lcov
retention-days: 1
coverage_report:
name: Coverage Report
runs-on: ubuntu-latest
timeout-minutes: 15
needs:
- find-changes
- rust_test_ubuntu
if: ${{ needs.find-changes.outputs.is-release-pr != 'true' && needs.find-changes.outputs.rust == 'true' }}
env:
COVERAGE_API_URL: ${{ secrets.COVERAGE_API_URL }}
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Setup Node
uses: actions/setup-node@v4
with:
node-version: "20"
- name: Download coverage artifacts
uses: actions/download-artifact@v4
with:
pattern: coverage-partition-*
merge-multiple: true
- name: Install lcov
run: sudo apt-get install -y lcov
- name: Merge and transform coverage
if: ${{ !github.event.pull_request.head.repo.fork }}
run: |
SAFE_BRANCH=$(echo "$BRANCH" | sed 's/[^a-zA-Z0-9_-]/_/g')
lcov --add-tracefile coverage-1.lcov --add-tracefile coverage-2.lcov --output-file coverage-merged.lcov
node scripts/transform-coverage.js coverage-merged.lcov coverage-transformed.json >> "$GITHUB_ENV"
npx vercel blob put coverage-transformed.json --pathname="coverage/commits/${SHA}.json" \
--token "${BLOB_READ_WRITE_TOKEN}"
npx vercel blob put coverage-transformed.json --pathname="coverage/branches/${SAFE_BRANCH}/${TIMESTAMP}.json" \
--token "${BLOB_READ_WRITE_TOKEN}"
echo "Uploaded coverage for ${SHA} on branch ${BRANCH}"
env:
SHA: ${{ github.sha }}
BRANCH: ${{ github.head_ref || github.ref_name }}
TIMESTAMP: ${{ github.event.head_commit.timestamp || github.event.pull_request.updated_at }}
BLOB_READ_WRITE_TOKEN: ${{ secrets.BLOB_READ_WRITE_TOKEN }}
- name: Comment on PR
if: github.event_name == 'pull_request' && env.COVERAGE_API_URL != '' && !github.event.pull_request.head.repo.fork
uses: actions/github-script@v7
with:
script: |
const lines = parseFloat(process.env.COVERAGE_LINES) || 0;
const functions = parseFloat(process.env.COVERAGE_FUNCTIONS) || 0;
const branches = parseFloat(process.env.COVERAGE_BRANCHES) || 0;
const body = `## Coverage Report
| Metric | Coverage |
|--------|----------|
| Lines | ${lines.toFixed(2)}% |
| Functions | ${functions.toFixed(2)}% |
| Branches | ${branches.toFixed(2)}% |
[View full report](${process.env.COVERAGE_API_URL}/commits/${{ github.sha }})
`;
const { data: comments } = await github.rest.issues.listComments({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.issue.number,
});
const botComment = comments.find(c =>
c.user.type === 'Bot' && c.body.includes('## Coverage Report')
);
if (botComment) {
await github.rest.issues.updateComment({
owner: context.repo.owner,
repo: context.repo.repo,
comment_id: botComment.id,
body,
});
} else {
await github.rest.issues.createComment({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.issue.number,
body,
});
}
check-examples:
name: Check examples
timeout-minutes: 40
needs:
- find-changes
if: ${{ needs.find-changes.outputs.is-release-pr != 'true' }}
runs-on: ubuntu-latest
env:
TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
TURBO_TEAM: ${{ vars.TURBO_TEAM }}
TURBO_CACHE: remote:rw
VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }}
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Setup Node.js
uses: actions/setup-node@v4
with:
node-version: "20"
- name: Setup pnpm
uses: pnpm/action-setup@v4
- name: Install dependencies
run: pnpm install
- name: Install Global Turbo
uses: ./.github/actions/install-global-turbo
- name: Setup Vercel credentials
run: |
cd examples
npx vercel link --yes --token=${{ secrets.VERCEL_TOKEN }} --project turborepo-examples-app
npx vercel env pull --yes --token=${{ secrets.VERCEL_TOKEN }}
- name: Run examples check
run: turbo run check-examples --filter=turborepo-examples --env-mode=strict
check-lockfiles:
name: Integration - Lockfile parsers
timeout-minutes: 15
needs:
- find-changes
if: ${{ needs.find-changes.outputs.is-release-pr != 'true' }}
runs-on: ubuntu-latest
env:
TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
TURBO_TEAM: ${{ vars.TURBO_TEAM }}
TURBO_CACHE: remote:rw
SCCACHE_BUCKET: turborepo-sccache
SCCACHE_REGION: us-east-2
RUSTC_WRAPPER: ${{ !github.event.pull_request.head.repo.fork && 'sccache' || '' }}
CARGO_INCREMENTAL: 0
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Setup Environment
uses: ./.github/actions/setup-environment
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
- name: Install Bun
uses: oven-sh/setup-bun@4bc047ad259df6fc24a6c9b0f9a0cb08cf17fbe5 # v2.0.1
- name: Run sccache-cache
uses: mozilla-actions/sccache-action@v0.0.6
- name: Build turbo binary
run: cargo build
- name: Run lockfile tests
run: pnpm check-lockfiles
working-directory: lockfile-tests
js_native_packages:
name: "@turbo/repository (${{matrix.os.name}}, Node ${{matrix.node-version}})"
needs:
- find-changes
if: ${{ needs.find-changes.outputs.is-release-pr != 'true' && needs.find-changes.outputs.native-lib == 'true' }}
timeout-minutes: 30
runs-on: ${{ matrix.os.runner }}
strategy:
fail-fast: false
matrix:
os:
- name: ubuntu
runner: ubuntu-latest
- name: macos
runner: macos-latest
node-version:
- 18
- 20
- 22
env:
TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
TURBO_TEAM: ${{ vars.TURBO_TEAM }}
TURBO_CACHE: remote:rw
steps:
- name: Determine fetch depth
id: fetch-depth
run: |
echo "depth=$(( ${{ github.event.pull_request.commits || 1 }} + 1 ))" >> $GITHUB_OUTPUT
- name: Checkout
uses: actions/checkout@v4
with:
ref: ${{ github.ref }}
fetch-depth: ${{ steps.fetch-depth.outputs.depth }}
- name: Setup Environment
uses: ./.github/actions/setup-environment
with:
github-token: "${{ secrets.GITHUB_TOKEN }}"
node-version: ${{ matrix.node-version }}
- name: Install Global Turbo
uses: ./.github/actions/install-global-turbo
- name: Run tests
run: |
TURBO_API= turbo run test --filter "turborepo-repository" --color --env-mode=strict
env:
NODE_VERSION: ${{ matrix.node-version }}
summary:
name: Test Summary
runs-on: ubuntu-latest
timeout-minutes: 30
if: always()
needs:
- find-changes
- integration
- turbo_types_check
- rust_test
- rust_test_ubuntu
- coverage_report
- check-examples
- check-lockfiles
- js_native_packages
steps:
- name: Skip for release PR
if: needs.find-changes.outputs.is-release-pr == 'true'
run: |
echo "Release PR detected - skipping tests (code already tested on main)"
- name: Compute info
id: info
if: needs.find-changes.outputs.is-release-pr != 'true'
run: |
cancelled=false
failure=false
subjob () {
local result=$1
if [ "$result" = "cancelled" ]; then
cancelled=true
elif [ "$result" != "success" ] && [ "$result" != "skipped" ]; then
failure=true
fi
}
subjob ${{needs.integration.result}}
subjob ${{needs.turbo_types_check.result}}
subjob ${{needs.rust_test.result}}
subjob ${{needs.rust_test_ubuntu.result}}
subjob ${{needs.coverage_report.result}}
subjob ${{needs.check-examples.result}}
subjob ${{needs.check-lockfiles.result}}
subjob ${{needs.js_native_packages.result}}
if [ "$cancelled" = "true" ]; then
echo "cancelled=true" >> $GITHUB_OUTPUT
elif [ "$failure" = "true" ]; then
echo "failure=true" >> $GITHUB_OUTPUT
else
echo "success=true" >> $GITHUB_OUTPUT
fi
- name: Failed
if: needs.find-changes.outputs.is-release-pr != 'true' && steps.info.outputs.failure == 'true'
run: exit 1
- name: Succeeded
if: needs.find-changes.outputs.is-release-pr != 'true' && steps.info.outputs.success == 'true'
run: echo Ok
cleanup:
name: Cleanup
needs: summary
if: always()
uses: ./.github/workflows/pr-clean-caches.yml
secrets: inherit