Refactor CI so that each platform downloads the source separately

Author: cuberoot

.github/workflows/dawn.yaml

@@ -26,7 +26,7 @@ env:
   DAWN_CONFIG: ${{ github.event.inputs.config || 'release' }}
 
 jobs:
-  get-dawn-source:
+  get-dawn-version:
     runs-on: ubuntu-latest
     outputs:
       chromium_channel: ${{ steps.read-version.outputs.chromium_channel }}
@@ -48,21 +48,10 @@ jobs:
           cd Dawn
           pip install -r requirements.txt
 
-      - name: Get Dawn source
+      - name: Get Dawn version
         run: |
           cd Dawn
-          python ci_build_dawn.py get-dawn --channel ${{ env.DAWN_CHANNEL }}
-
-      - name: Upload Dawn source
-        uses: actions/upload-artifact@v4
-        with:
-          name: dawn-source
-          path: |
-            Dawn/dawn_source/
-            !Dawn/dawn_source/.git
-            !Dawn/dawn_source/**/*.git
-            !Dawn/dawn_source/third_party/khronos/OpenGL-Registry/specs
-          retention-days: 1
+          python ci_build_dawn.py get-dawn-version --channel ${{ env.DAWN_CHANNEL }}
 
       - name: Upload Dawn Version
         uses: actions/upload-artifact@v4
@@ -85,7 +74,7 @@ jobs:
           echo "chromium_dawn_suffix=$value" >> $GITHUB_OUTPUT
 
   build-dawn:
-    needs: get-dawn-source
+    needs: get-dawn-version
     strategy:
       matrix:
         include:
@@ -129,10 +118,9 @@ jobs:
           pip install -r requirements.txt
 
       - name: Download Dawn source
-        uses: actions/download-artifact@v5
-        with:
-          name: dawn-source
-          path: Dawn/dawn_source/
+        run: |
+          cd Dawn
+          python ci_build_dawn.py get-source --hash ${{ needs.get-dawn-version.outputs.chromium_dawn_hash }}
 
       - name: Build Dawn for ${{ matrix.target }}
         run: |
@@ -148,8 +136,16 @@ jobs:
             !Dawn/builds/*/out
           retention-days: 1
 
+      - name: Upload dawn.json
+        if: matrix.platform == 'ubuntu-latest'
+        uses: actions/upload-artifact@v4
+        with:
+          name: dawn-json
+          path: Dawn/dawn_source/src/dawn/dawn.json
+          retention-days: 1
+
   create-bundle:
-    needs: [build-dawn, get-dawn-source]
+    needs: [build-dawn, get-dawn-version]
     runs-on: ubuntu-latest
     outputs:
       bundle_name: ${{ steps.set-bundle-name.outputs.bundle_name }}
@@ -168,11 +164,11 @@ jobs:
           cd Dawn
           pip install -r requirements.txt
 
-      - name: Download Dawn source
+      - name: Download dawn.json
         uses: actions/download-artifact@v5
         with:
-          name: dawn-source
-          path: Dawn/dawn_source/
+          name: dawn-json
+          path: Dawn/dawn_source/src/dawn
 
       - name: Download all build artifacts
         uses: actions/download-artifact@v5
@@ -196,7 +192,7 @@ jobs:
       - name: Set bundle name
         id: set-bundle-name
         run: |
-          BUNDLE_NAME="dawn_webgpu_${{needs.get-dawn-source.outputs.chromium_dawn_suffix}}.zip"
+          BUNDLE_NAME="dawn_webgpu_${{needs.get-dawn-version.outputs.chromium_dawn_suffix}}.zip"
           echo "bundle_name=$BUNDLE_NAME" >> $GITHUB_OUTPUT
 
       - name: Upload bundle artifact
@@ -207,7 +203,7 @@ jobs:
           retention-days: 30
 
   create-release:
-    needs: [create-bundle, get-dawn-source]
+    needs: [create-bundle, get-dawn-version]
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
@@ -230,19 +226,19 @@ jobs:
       - name: Tag name
         id: tag
         run: |
-          TAG="dawn-chromium-${{env.DAWN_CHANNEL}}-${{needs.get-dawn-source.outputs.chromium_dawn_version}}"
+          TAG="dawn-chromium-${{env.DAWN_CHANNEL}}-${{needs.get-dawn-version.outputs.chromium_dawn_version}}"
           echo "tag=$TAG" >> $GITHUB_OUTPUT
 
       - name: Create Release
         uses: actions/create-release@v1
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
-          release_name: Dawn WebGPU from Chromium ${{ needs.get-dawn-source.outputs.chromium_dawn_version }}
+          release_name: Dawn WebGPU from Chromium ${{ needs.get-dawn-version.outputs.chromium_dawn_version }}
           tag_name: ${{ steps.tag.outputs.tag }}
           body: |
-            Dawn WebGPU build matching Chromium ${{ needs.get-dawn-source.outputs.chromium_dawn_version }}.
-            Built from Dawn hash: ${{ needs.get-dawn-source.outputs.chromium_dawn_hash }}
+            Dawn WebGPU build matching Chromium ${{ needs.get-dawn-version.outputs.chromium_dawn_version }}.
+            Built from Dawn hash: ${{ needs.get-dawn-version.outputs.chromium_dawn_hash }}
             SHA256: ${{ steps.compute-sha256.outputs.sha256sum }}
 
             This release contains pre-built Dawn WebGPU libraries for multiple platforms:

Dawn/README.md

@@ -21,8 +21,12 @@ If you wish to build a version of Dawn yourself, you can use these CI scripts as
 python3 -m venv .venv
 pip3 install -r requirements.txt
 
-# Download the Dawn source matching the latest release of Chromium Canary
-./ci_build_dawn.py get-dawn
+# Determine the latest release of Chromium Canary and get the Dawn hash
+# Write the data to dawn_version.json
+./ci_build_dawn.py get-dawn-version
+
+# Download the Dawn source matching the given hash
+./ci_build_dawn.py get-source --hash ABC1234ABC
 
 # Build Dawn, running these commands on the appropriate platform
 # Note that macosx builds both Intel and Arm

Dawn/ci_build_dawn.py

@@ -77,14 +77,23 @@ def parse_args() -> argparse.Namespace:
     parser = argparse.ArgumentParser(description="CI build tools for Dawn")
     subparsers = parser.add_subparsers(dest="command")
 
-    get_parser = subparsers.add_parser("get-dawn", help="Get the latest Dawn source")
+    get_parser = subparsers.add_parser(
+        "get-dawn-version", help="Get the latest Dawn version"
+    )
     get_parser.add_argument(
         "--channel",
         choices=["stable", "beta", "canary"],
         default="canary",
         help="Chromium channel to use for Dawn",
     )
 
+    get_parser = subparsers.add_parser("get-source", help="Get the Dawn source")
+    get_parser.add_argument(
+        "--hash",
+        required=True,
+        help="Dawn hash to get the source for",
+    )
+
     build_parser = subparsers.add_parser("build-target", help="Build a target")
     build_parser.add_argument(
         "--target",
@@ -114,8 +123,11 @@ def main() -> int:
         Exit code (0 for success, 1 for failure)
     """
     args = parse_args()
-    if args.command == "get-dawn":
+    if args.command == "get-dawn-version":
         dawn_source.get_matching_dawn_for_chromium(args.channel)
+        print(dawn_source.get_version())
+    elif args.command == "get-source":
+        dawn_source.fetch_dawn_source(args.hash)
     elif args.command == "build-target":
         if args.target not in [
             "windows",

Dawn/dawn_source.py

@@ -34,12 +34,10 @@ class DawnSourceToolsDirectoryNotFoundError(DawnSourceDirectoryConfigurationErro
 def get_matching_dawn_for_chromium(channel: str = "canary") -> None:
     """
     Get the Dawn version that matches the latest Chromium release for the given channel
+    and write the version data to the dawn_version.json file.
 
     Args:
         channel: The Chromium channel to use for fetching the latest version
-
-    Returns:
-        A tuple containing the Dawn hash, version, and suffix
     """
 
     chromium_dawn_hash, chromium_dawn_version, chromium_dawn_suffix = (