[Converter] Time manifest processing operations; Remove logic to go through manifests in all snapshots causing unnecessary latency. #1655
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| pull_request: | |
| types: | |
| - opened | |
| - edited | |
| - synchronize | |
| - reopened | |
| branches: [main, phash_main, '2.0'] | |
| jobs: | |
| lint: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@v4 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v7 | |
| with: | |
| python-version: "3.9" | |
| - name: Install dependencies | |
| run: uv sync --all-extras | |
| - name: Linting | |
| run: uv run pre-commit run --all-files | |
| build-n-test: | |
| name: Build and test (Daft native runner) | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| python-version: ["3.9", "3.10"] | |
| timeout-minutes: 45 | |
| steps: | |
| - name: "checkout repository" | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v7 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install dependencies | |
| run: uv sync --all-extras | |
| - name: Run unit tests + benchmarks (Daft native runner) | |
| run: uv run pytest -m "not distributed and not integration" --benchmark-json output.json | |
| env: | |
| DAFT_RUNNER: native | |
| # Download previous benchmark result from cache (if exists) | |
| - name: Download previous benchmark data | |
| uses: actions/cache@v4 | |
| with: | |
| path: ./cache | |
| key: ${{ runner.os }}-benchmark | |
| - name: Store benchmark results | |
| uses: benchmark-action/github-action-benchmark@v1 | |
| with: | |
| tool: "pytest" | |
| output-file-path: output.json | |
| auto-push: false | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| # Where the previous data file is stored | |
| external-data-json-path: ./cache/benchmark-data.json | |
| # Enable Job Summary for PRs | |
| summary-always: true | |
| distributed-tests: | |
| name: Build and distributed test (Daft Ray runner) | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| python-version: ["3.9", "3.10"] | |
| daft-runner: [ray] | |
| timeout-minutes: 45 | |
| steps: | |
| - name: "checkout repository" | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v7 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install dependencies | |
| run: uv sync --all-extras | |
| - name: Run distributed tests (Daft distributed runner) | |
| run: uv run pytest -m "distributed" --tb=short | |
| env: | |
| DAFT_RUNNER: ${{ matrix.daft-runner }} |