chore(ingestion): Bump urllib3 floor (#19056) #13
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Writes the master-scoped dependency cache entries that PR jobs restore from. | |
| # | |
| # Why this exists: Actions caches are ref-scoped — a PR run can read caches written | |
| # at refs/heads/master, but a cache saved from a PR ref is visible only to that PR. | |
| # lint-jobs.yml is `on: pull_request` only, so nothing in it can ever write a | |
| # master-scoped entry; when its jobs saved their own caches, every active PR minted | |
| # a private copy of the same key (several hundred MB each) against the repo's 10GB | |
| # Actions cache budget, and the shared master entries were evicted first. | |
| # | |
| # The yarn jobs exist for a related reason: build-and-test.yml's frontend jobs run | |
| # on Depot runners (#18998), whose actions/cache writes go to Depot's own backend — | |
| # GitHub-hosted lint jobs cannot read them, so without a GitHub-hosted writer the | |
| # master-scoped yarn entries would age out and never be replaced. | |
| # | |
| # This workflow runs on every push to master. When the exact key already exists the | |
| # lookup-only restore hits and the job exits in seconds; on a miss (dependency | |
| # manifests changed, or the entry was evicted) it rebuilds the environments and | |
| # saves one master-scoped copy. Keys and cache paths must stay byte-identical to | |
| # the restore steps in lint-jobs.yml. | |
| name: CI Cache Warmup | |
| on: | |
| push: | |
| branches: | |
| - master | |
| workflow_dispatch: | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| warm-python-lint-cache: | |
| name: Warm python-lint uv cache | |
| # Master-gated so a workflow_dispatch from a branch or tag no-ops instead of | |
| # saving a ref-scoped copy nothing can read. | |
| if: github.ref == 'refs/heads/master' | |
| # Must be a GitHub-hosted runner: the whole point is writing to the GitHub | |
| # Actions cache. Depot runners use Depot Cache, which GitHub-hosted PR jobs | |
| # cannot read. | |
| runs-on: ubuntu-latest | |
| env: | |
| UV_CACHE_DIR: /tmp/.uv-cache/ | |
| DATAHUB_TELEMETRY_ENABLED: false | |
| steps: | |
| - name: Check out the repo | |
| uses: acryldata/sane-checkout-action@186e92cc5948a9c3e1cc7a96eaff9f776f3fc8e3 # v7 | |
| # Key must stay byte-identical with the restore key in lint-jobs.yml python-lint | |
| # and the save key at the bottom of this job. | |
| - name: Check for existing cache entry | |
| id: uv-cache | |
| uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5 | |
| with: | |
| path: ${{ env.UV_CACHE_DIR }} | |
| key: uv-python-lint-${{ hashFiles('./metadata-ingestion/setup.py', './metadata-ingestion/pyproject.toml', './metadata-ingestion/uv.lock', './metadata-ingestion/constraints.txt', './metadata-ingestion/build-constraints.txt', './datahub-actions/setup.py', './datahub-actions/pyproject.toml', './datahub-agent-context/setup.py', './datahub-agent-context/pyproject.toml', './metadata-ingestion-modules/airflow-plugin/setup.py', './metadata-ingestion-modules/airflow-plugin/pyproject.toml', './metadata-ingestion-modules/gx-plugin/setup.py', './metadata-ingestion-modules/gx-plugin/pyproject.toml', './metadata-ingestion-modules/dagster-plugin/setup.py', './metadata-ingestion-modules/dagster-plugin/pyproject.toml', './metadata-ingestion-modules/prefect-plugin/setup.py', './metadata-ingestion-modules/prefect-plugin/pyproject.toml') }} | |
| lookup-only: true | |
| - name: Set up JDK 21 | |
| if: steps.uv-cache.outputs.cache-hit != 'true' | |
| uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5 | |
| with: | |
| distribution: "zulu" | |
| java-version: 21 | |
| - uses: gradle/actions/setup-gradle@0723195856401067f7a2779048b490ace7a47d7c # v5.0.2 | |
| if: steps.uv-cache.outputs.cache-hit != 'true' | |
| - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 | |
| if: steps.uv-cache.outputs.cache-hit != 'true' | |
| with: | |
| python-version: "3.11" | |
| - name: Install system dependencies | |
| if: steps.uv-cache.outputs.cache-hit != 'true' | |
| run: ./metadata-ingestion/scripts/install_deps.sh | |
| # installDev is what the lint tasks in lint-jobs.yml python-lint depend on; running | |
| # it for every module populates the same uv download/wheel cache the lint jobs need. | |
| - name: Install Python module dev environments | |
| if: steps.uv-cache.outputs.cache-hit != 'true' | |
| run: | | |
| ./gradlew :metadata-ingestion:installDev \ | |
| :datahub-actions:installDev \ | |
| :datahub-agent-context:installDev \ | |
| :metadata-ingestion-modules:airflow-plugin:installDev \ | |
| :metadata-ingestion-modules:gx-plugin:installDev \ | |
| :metadata-ingestion-modules:dagster-plugin:installDev \ | |
| :metadata-ingestion-modules:prefect-plugin:installDev | |
| # No `always()`: a partial cache from a failed install would occupy the exact key | |
| # until the manifests next change. On failure, the next master push retries. | |
| - name: Save uv cache | |
| if: steps.uv-cache.outputs.cache-hit != 'true' | |
| uses: actions/cache/save@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5 | |
| with: | |
| path: ${{ env.UV_CACHE_DIR }} | |
| key: uv-python-lint-${{ hashFiles('./metadata-ingestion/setup.py', './metadata-ingestion/pyproject.toml', './metadata-ingestion/uv.lock', './metadata-ingestion/constraints.txt', './metadata-ingestion/build-constraints.txt', './datahub-actions/setup.py', './datahub-actions/pyproject.toml', './datahub-agent-context/setup.py', './datahub-agent-context/pyproject.toml', './metadata-ingestion-modules/airflow-plugin/setup.py', './metadata-ingestion-modules/airflow-plugin/pyproject.toml', './metadata-ingestion-modules/gx-plugin/setup.py', './metadata-ingestion-modules/gx-plugin/pyproject.toml', './metadata-ingestion-modules/dagster-plugin/setup.py', './metadata-ingestion-modules/dagster-plugin/pyproject.toml', './metadata-ingestion-modules/prefect-plugin/setup.py', './metadata-ingestion-modules/prefect-plugin/pyproject.toml') }} | |
| # Separate job (not extra steps above) so the saved entry contains only the | |
| # smoke-test dependencies, not the union with the python-lint cache. | |
| # smoke-test's requirements.txt installs '-e ../metadata-ingestion[...]', so | |
| # metadata-ingestion's manifests are install inputs for this key too. | |
| warm-smoke-test-lint-cache: | |
| name: Warm smoke-test lint uv cache | |
| if: github.ref == 'refs/heads/master' | |
| runs-on: ubuntu-latest | |
| env: | |
| UV_CACHE_DIR: /tmp/.uv-cache/ | |
| DATAHUB_TELEMETRY_ENABLED: false | |
| steps: | |
| - name: Check out the repo | |
| uses: acryldata/sane-checkout-action@186e92cc5948a9c3e1cc7a96eaff9f776f3fc8e3 # v7 | |
| # Key must stay byte-identical with the restore key in lint-jobs.yml | |
| # smoke-test-lint and the save key at the bottom of this job. | |
| - name: Check for existing cache entry | |
| id: uv-cache | |
| uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5 | |
| with: | |
| path: ${{ env.UV_CACHE_DIR }} | |
| key: "uv-smoke-test-lint-${{ hashFiles('./smoke-test/requirements.txt', './smoke-test/pyproject.toml', './metadata-ingestion/setup.py', './metadata-ingestion/pyproject.toml') }}" | |
| lookup-only: true | |
| - name: Set up JDK 21 | |
| if: steps.uv-cache.outputs.cache-hit != 'true' | |
| uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5 | |
| with: | |
| distribution: "zulu" | |
| java-version: 21 | |
| - uses: gradle/actions/setup-gradle@0723195856401067f7a2779048b490ace7a47d7c # v5.0.2 | |
| if: steps.uv-cache.outputs.cache-hit != 'true' | |
| - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 | |
| if: steps.uv-cache.outputs.cache-hit != 'true' | |
| with: | |
| python-version: "3.11" | |
| - name: Install smoke-test dev environment | |
| if: steps.uv-cache.outputs.cache-hit != 'true' | |
| run: ./gradlew :smoke-test:installDev | |
| - name: Save uv cache | |
| if: steps.uv-cache.outputs.cache-hit != 'true' | |
| uses: actions/cache/save@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5 | |
| with: | |
| path: ${{ env.UV_CACHE_DIR }} | |
| key: "uv-smoke-test-lint-${{ hashFiles('./smoke-test/requirements.txt', './smoke-test/pyproject.toml', './metadata-ingestion/setup.py', './metadata-ingestion/pyproject.toml') }}" | |
| # GitHub-hosted writer for the shared datahub-web-react yarn cache. Restored by | |
| # lint-jobs.yml (datahub-web-react-lint, markdown-format, github-actions-format) | |
| # and by build-and-test.yml's frontend jobs when they run GitHub-hosted. | |
| warm-web-yarn-cache: | |
| name: Warm datahub-web-react yarn cache | |
| if: github.ref == 'refs/heads/master' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check out the repo | |
| uses: acryldata/sane-checkout-action@186e92cc5948a9c3e1cc7a96eaff9f776f3fc8e3 # v7 | |
| # Key must stay byte-identical with the yarn restore keys in lint-jobs.yml and | |
| # the yarn cache key in build-and-test.yml frontend-build. | |
| - name: Check for existing cache entry | |
| id: yarn-cache | |
| uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5 | |
| with: | |
| path: ~/.cache/yarn | |
| key: ${{ runner.os }}-yarn-${{ hashFiles('datahub-web-react/yarn.lock') }} | |
| lookup-only: true | |
| - name: Set up JDK 21 | |
| if: steps.yarn-cache.outputs.cache-hit != 'true' | |
| uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5 | |
| with: | |
| distribution: "zulu" | |
| java-version: 21 | |
| - uses: gradle/actions/setup-gradle@0723195856401067f7a2779048b490ace7a47d7c # v5.0.2 | |
| if: steps.yarn-cache.outputs.cache-hit != 'true' | |
| - name: Populate yarn cache | |
| if: steps.yarn-cache.outputs.cache-hit != 'true' | |
| run: ./gradlew :datahub-web-react:yarnInstall | |
| - name: Save yarn cache | |
| if: steps.yarn-cache.outputs.cache-hit != 'true' | |
| uses: actions/cache/save@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5 | |
| with: | |
| path: ~/.cache/yarn | |
| key: ${{ runner.os }}-yarn-${{ hashFiles('datahub-web-react/yarn.lock') }} | |
| # Separate job so the playwright entry stays small instead of duplicating the | |
| # ~1.2GB web-react cache under a second key. Uses the 'playwright-<os>-yarn-' | |
| # family shared with resusable-playwright-tests.yml — that workflow's own | |
| # master-gated save runs on Depot runners on master, whose cache writes | |
| # GitHub-hosted jobs cannot read, so this job is the family's GitHub-hosted | |
| # writer. The distinct prefix also keeps this small entry out of the | |
| # '<os>-yarn-' family, so a web-cache prefix restore can never land on it. | |
| warm-playwright-yarn-cache: | |
| name: Warm playwright lint yarn cache | |
| if: github.ref == 'refs/heads/master' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check out the repo | |
| uses: acryldata/sane-checkout-action@186e92cc5948a9c3e1cc7a96eaff9f776f3fc8e3 # v7 | |
| # Key must stay byte-identical with the yarn restore key in lint-jobs.yml | |
| # playwright-e2e-lint and the cache keys in resusable-playwright-tests.yml. | |
| - name: Check for existing cache entry | |
| id: yarn-cache | |
| uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5 | |
| with: | |
| path: ~/.cache/yarn | |
| key: playwright-${{ runner.os }}-yarn-${{ hashFiles('./e2e-test/ui/playwright/package.json', './e2e-test/ui/playwright/yarn.lock') }} | |
| lookup-only: true | |
| - name: Set up Node.js | |
| if: steps.yarn-cache.outputs.cache-hit != 'true' | |
| uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 | |
| with: | |
| node-version: "22" | |
| - name: Populate yarn cache | |
| if: steps.yarn-cache.outputs.cache-hit != 'true' | |
| working-directory: e2e-test/ui/playwright | |
| run: yarn install --frozen-lockfile | |
| - name: Save yarn cache | |
| if: steps.yarn-cache.outputs.cache-hit != 'true' | |
| uses: actions/cache/save@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5 | |
| with: | |
| path: ~/.cache/yarn | |
| key: playwright-${{ runner.os }}-yarn-${{ hashFiles('./e2e-test/ui/playwright/package.json', './e2e-test/ui/playwright/yarn.lock') }} |