Cache Generator #9
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # This workflow is the single source of truth for generating and updating various caches used by our CI pipeline. | |
| # It sets the fresh cache each Monday but provides a manual trigger to update it on-demand. | |
| # | |
| # All other jobs only restore from this cache; *they never update it*! | |
| # They should always use `actions/cache/restore@v4` because `actions/cache@v4` would attempt to save the cache. | |
| # | |
| # Our agreed-upon strategy is to use a deterministic cache key based on the week of the year. | |
| # Jobs can utilize the prior week's cache if the current week's cache is not found. | |
| # | |
| # | |
| # PRINCIPLES: | |
| # - The cache is never wrong! | |
| # - Caches are immutable once created. | |
| # - Caches are for speeding up builds, not for correctness. | |
| # - It is faster to download a large cache than to re-resolve missing dependencies. | |
| # | |
| # | |
| # DESIGN: | |
| # | |
| # 1. Prepare | |
| # - A job first prepares the local environment (e.g., installs JDK, Clojure, Node.js, etc.) | |
| # - This fetches the dependencies from the cache (if it exists) and warms up the local cache for this workflow. | |
| # 2. Resolve | |
| # - The next step pre-resolves all dependencies, ensuring that the local cache is fully populated and up-to-date. | |
| # 3. Update | |
| # - The final step saves the local cache back to GitHub Actions cache with a deterministic key. | |
| # - If a cache entry already exists for that key, it is deleted first to ensure immutability. | |
| # | |
| name: Cache Generator | |
| on: | |
| schedule: | |
| - cron: "0 3 * * 1" # Mondays at 03:00 UTC | |
| workflow_dispatch: # manually update the cache | |
| concurrency: | |
| group: cache-generator | |
| cancel-in-progress: true | |
| jobs: | |
| get-cache-keys: | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Get cache keys | |
| uses: ./.github/actions/get-cache-keys | |
| id: get-cache-keys | |
| outputs: | |
| m2-cache-key: ${{ steps.get-cache-keys.outputs.m2-cache-key }} | |
| node-cache-key: ${{ steps.get-cache-keys.outputs.node-cache-key }} | |
| eslint-cache-key: ${{ steps.get-cache-keys.outputs.eslint-cache-key }} | |
| cypress-cache-key: ${{ steps.get-cache-keys.outputs.cypress-cache-key }} | |
| generate-caches: | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 30 | |
| needs: get-cache-keys | |
| env: | |
| M2_CACHE_KEY: ${{ needs.get-cache-keys.outputs.m2-cache-key }} | |
| NODE_CACHE_KEY: ${{ needs.get-cache-keys.outputs.node-cache-key }} | |
| CYPRESS_CACHE_KEY: ${{ needs.get-cache-keys.outputs.cypress-cache-key }} | |
| ESLINT_CACHE_KEY: ${{ needs.get-cache-keys.outputs.eslint-cache-key }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| # Backend preparation and cache | |
| - name: Prepare backend | |
| uses: ./.github/actions/prepare-backend | |
| # Add or remove aliases to affect the cache size and its contents. | |
| # Adjust to taste, then redeploy this workflow. | |
| - name: Pre-resolve Clojure dependencies | |
| run: clojure -A:build:dev:ee:ee-dev:drivers:drivers-dev:cljs -P | |
| - name: Update M2 cache | |
| uses: ./.github/actions/update-cache | |
| with: | |
| path: | | |
| ~/.m2 | |
| ~/.gitlibs | |
| key: ${{ env.M2_CACHE_KEY }} | |
| # Frontend preparation and caches | |
| # `prepare-frontend` action (1) prepares Node.js and (2) fetches the node_modules cache if it exists. | |
| # It then (3) runs `yarn install --frozen-lockfile --prefer-offline` which ensures the local cache is fully populated. | |
| # IMPORTANT: Caching node_modules after patches are already applied will break any PR that attempts to either: | |
| # - upgrade a patched library | |
| # - change a patch | |
| # Hence why we explicitly exclude patches from the cache (controlled by the `apply-patches` action input). | |
| - name: Install front-end dependencies | |
| uses: ./.github/actions/prepare-frontend | |
| with: | |
| apply-patches: "false" | |
| - name: Get yarn cache directory | |
| id: yarn-cache-dir | |
| run: | | |
| YARN_CACHE_DIR=$(yarn cache dir) | |
| echo "Yarn cache directory: $YARN_CACHE_DIR" | |
| echo "dir=$YARN_CACHE_DIR" >> $GITHUB_OUTPUT | |
| - name: Ensure that Cypress executable is ready | |
| run: | | |
| yarn cypress install | |
| yarn cypress cache path | |
| yarn cypress cache list | |
| yarn cypress verify | |
| shell: bash | |
| # Run Prettier to populate its cache (stored in node_modules/.cache/prettier) | |
| # so it gets saved as part of the node_modules cache below | |
| - name: Run Prettier to populate cache | |
| run: yarn lint-prettier-pure | |
| - name: Update node_modules cache | |
| uses: ./.github/actions/update-cache | |
| with: | |
| path: | | |
| ${{ steps.yarn-cache-dir.outputs.dir }} | |
| node_modules | |
| key: ${{ env.NODE_CACHE_KEY }} | |
| - name: Update Cypress cache | |
| uses: ./.github/actions/update-cache | |
| with: | |
| path: ~/.cache/Cypress | |
| key: ${{ env.CYPRESS_CACHE_KEY }} | |
| # ESLint cache (requires both backend and frontend) | |
| - name: Compile CLJS | |
| run: yarn build-pure:cljs | |
| - name: Run ESLint to populate cache | |
| run: yarn lint-eslint-pure | |
| - name: Update eslint cache | |
| uses: ./.github/actions/update-cache | |
| with: | |
| path: ${{ github.workspace}}/.eslintcache | |
| key: ${{ env.ESLINT_CACHE_KEY }} |