Split dagster-cloud.pex by runner architecture (#247)

gibsondan · claude · web-flow · commit c1051c63fc86 · 2026-05-22T15:38:36.000-05:00
split dagster-cloud.pex by runner arch

Build two PEX variants — dagster-cloud-x86_64.pex (x86 runners +
manylinux docker, ~80MB) and dagster-cloud-aarch64.pex (ARM runners,
~50MB) — instead of one ~96MB fat PEX. Keeps every file comfortably
under GitHub's 100MB per-file limit. action.yml call sites route by
runner.arch; src/Dockerfile pins to the x86_64 variant; deploy_pex.py
picks by platform.machine() at runtime.

generated/gha/dagster-cloud.pex becomes a symlink to the x86_64 variant
for one release so external consumers reaching past the public action
API continue to work on x86 runners.

Co-authored-by: Claude Opus 4.7 (1M context) &lt;noreply@anthropic.com&gt;
diff --git a/actions/build_deploy_python_executable/action.yml b/actions/build_deploy_python_executable/action.yml
@@ -72,7 +72,7 @@ runs:
     - if: ${{ inputs.deploy != 'true' }}
       run: >
         cd $ACTION_REPO &&
-        generated/gha/dagster-cloud.pex -m dagster_cloud_cli.entrypoint
+        generated/gha/dagster-cloud-${{ runner.arch == 'ARM64' && 'aarch64' || 'x86_64' }}.pex -m dagster_cloud_cli.entrypoint
         serverless build-python-executable
         $SOURCE_DIRECTORY ${{ inputs.build_output_dir }}
         --python-version=${{ inputs.python_version }}
diff --git a/actions/utils/ci-init/action.yml b/actions/utils/ci-init/action.yml
@@ -21,7 +21,7 @@ runs:
   steps:
     - name: init-env-vars
       run: >
-        echo "DAGSTER_CLOUD_PEX=$GITHUB_ACTION_PATH/../../../generated/gha/dagster-cloud.pex" >> $GITHUB_ENV &&
+        echo "DAGSTER_CLOUD_PEX=$GITHUB_ACTION_PATH/../../../generated/gha/dagster-cloud-${{ runner.arch == 'ARM64' && 'aarch64' || 'x86_64' }}.pex" >> $GITHUB_ENV &&
         echo "DAGSTER_BUILD_STATEDIR=/tmp/statedir-$GITHUB_RUN_ID" >> $GITHUB_ENV
       shell: bash
 
diff --git a/actions/utils/dagster-cloud-cli/action.yml b/actions/utils/dagster-cloud-cli/action.yml
@@ -9,5 +9,5 @@ runs:
   using: "composite"
   steps:
     - id: dagster-cloud-cli
-      run: $GITHUB_ACTION_PATH/../../../generated/gha/dagster-cloud.pex -m dagster_cloud_cli.entrypoint ${{ inputs.command }}
+      run: $GITHUB_ACTION_PATH/../../../generated/gha/dagster-cloud-${{ runner.arch == 'ARM64' && 'aarch64' || 'x86_64' }}.pex -m dagster_cloud_cli.entrypoint ${{ inputs.command }}
       shell: bash
diff --git a/actions/utils/dg-cli/action.yml b/actions/utils/dg-cli/action.yml
@@ -9,5 +9,5 @@ runs:
   using: "composite"
   steps:
     - id: dg-cli
-      run: $GITHUB_ACTION_PATH/../../../generated/gha/dagster-cloud.pex -m dagster_dg_cli.cli.entrypoint ${{ inputs.command }}
+      run: $GITHUB_ACTION_PATH/../../../generated/gha/dagster-cloud-${{ runner.arch == 'ARM64' && 'aarch64' || 'x86_64' }}.pex -m dagster_dg_cli.cli.entrypoint ${{ inputs.command }}
       shell: bash
diff --git a/actions/utils/dg-deploy-init/action.yaml b/actions/utils/dg-deploy-init/action.yaml
@@ -17,7 +17,7 @@ runs:
   steps:
     - name: init-env-vars
       run: >
-        echo "DAGSTER_CLOUD_PEX=$GITHUB_ACTION_PATH/../../../generated/gha/dagster-cloud.pex" >> $GITHUB_ENV &&
+        echo "DAGSTER_CLOUD_PEX=$GITHUB_ACTION_PATH/../../../generated/gha/dagster-cloud-${{ runner.arch == 'ARM64' && 'aarch64' || 'x86_64' }}.pex" >> $GITHUB_ENV &&
         echo "DAGSTER_BUILD_STATEDIR=/tmp/statedir-$GITHUB_RUN_ID" >> $GITHUB_ENV
       shell: bash
 
diff --git a/actions/utils/prerun/action.yml b/actions/utils/prerun/action.yml
@@ -26,7 +26,7 @@ runs:
       # closed PRs, this marks the pr_status=closed and attaches the merge commit details. 
       run: >
         echo "::notice title=Closed Pull Request::Marking branch deployment closed for this PR, will skip remaining workflow" &&
-        $GITHUB_ACTION_PATH/../../../generated/gha/dagster-cloud.pex -m dagster_cloud_cli.entrypoint ci branch-deployment prerun_checkout_dir > /tmp/closed-branch-deployment.txt &&
+        $GITHUB_ACTION_PATH/../../../generated/gha/dagster-cloud-${{ runner.arch == 'ARM64' && 'aarch64' || 'x86_64' }}.pex -m dagster_cloud_cli.entrypoint ci branch-deployment prerun_checkout_dir > /tmp/closed-branch-deployment.txt &&
         echo "closed_branch_deployment=$(cat /tmp/closed-branch-deployment.txt)" >> "$GITHUB_OUTPUT" &&
         echo 'result=skip' >> "$GITHUB_OUTPUT"
       shell: bash
diff --git a/generated/gha/dagster-cloud-aarch64.pex b/generated/gha/dagster-cloud-aarch64.pex
diff --git a/generated/gha/dagster-cloud-x86_64.pex b/generated/gha/dagster-cloud-x86_64.pex
diff --git a/generated/gha/dagster-cloud.pex b/generated/gha/dagster-cloud.pex
diff --git a/generated/gha/dagster-cloud.pex b/generated/gha/dagster-cloud.pex
@@ -0,0 +1 @@
+dagster-cloud-x86_64.pex
diff --git a/scripts/release.py b/scripts/release.py
@@ -109,47 +109,69 @@ def build_dagster_cloud_pex(
         dagster_pipes_pkg = "dagster-pipes"
         dagster_shared_pkg = "dagster-shared"
 
-    complete_platform_args = []
-
-    for json_file in {
-        "x86_64_310.json",  # ubuntu-22.04 action runner
-        "x86_64_312.json",  # ubuntu-24.04 action runner
-        "aarch64_312.json",  # ubuntu-24.04-arm action runner
-        "manylinux_2_28_x86_64.json",  # used by the distributed Dockerfile
-    }:
-        with open(os.path.join(os.path.dirname(__file__), "complete_platforms", json_file)) as f:
-            complete_platform = f.read()
-            complete_platform_args.append(f"--complete-platform={complete_platform}")
-
-    info("Building generated/gha/dagster-cloud.pex")
-    args = [
-        "pex",
-        dagster_cloud_cli_pkg,
-        dagster_pkg,
-        dagster_dg_cli_pkg,
-        dagster_dg_core_pkg,
-        dagster_pipes_pkg,
-        dagster_shared_pkg,
-        "PyGithub",
-        "pex>=2.1.132,<3",
-        "pip",
-        "-o=dagster-cloud.pex",
-        *complete_platform_args,
-        "--pip-version=23.0",
-        "--resolver-version=pip-2020-resolver",
-        "--venv=prepend",
-        "--sh-boot",
-        "-vvvvv",
-    ]
-    print(f"Running {args}")
-    output = subprocess.check_output(
-        args,
-        shell=False,
-        encoding="utf-8",
-    )
-    print(output)
-    shutil.move("dagster-cloud.pex", "generated/gha/dagster-cloud.pex")
-    info("Built generated/gha/dagster-cloud.pex")
+    # Split the pex by runner architecture to stay well under GitHub's 100MB
+    # per-file limit. aarch64 wheels are native and don't overlap with x86,
+    # so removing them from the x86 PEX is the bulk of the win. The
+    # manylinux_2_28_x86_64 platform is bundled into the x86 variant since it
+    # overlaps heavily with the Ubuntu x86 runner wheels (~4MB extra).
+    variants = {
+        "dagster-cloud-x86_64.pex": [
+            "x86_64_310.json",  # ubuntu-22.04 action runner
+            "x86_64_312.json",  # ubuntu-24.04 action runner
+            "manylinux_2_28_x86_64.json",  # used by the distributed Dockerfile
+        ],
+        "dagster-cloud-aarch64.pex": [
+            "aarch64_312.json",  # ubuntu-24.04-arm action runner
+        ],
+    }
+
+    for output_name, platform_files in variants.items():
+        complete_platform_args = []
+        for json_file in platform_files:
+            with open(
+                os.path.join(os.path.dirname(__file__), "complete_platforms", json_file)
+            ) as f:
+                complete_platform = f.read()
+                complete_platform_args.append(f"--complete-platform={complete_platform}")
+
+        info(f"Building generated/gha/{output_name}")
+        args = [
+            "pex",
+            dagster_cloud_cli_pkg,
+            dagster_pkg,
+            dagster_dg_cli_pkg,
+            dagster_dg_core_pkg,
+            dagster_pipes_pkg,
+            dagster_shared_pkg,
+            "PyGithub",
+            "pex>=2.1.132,<3",
+            "pip",
+            f"-o={output_name}",
+            *complete_platform_args,
+            "--pip-version=23.0",
+            "--resolver-version=pip-2020-resolver",
+            "--venv=prepend",
+            "--sh-boot",
+            "-vvvvv",
+        ]
+        print(f"Running {args}")
+        output = subprocess.check_output(
+            args,
+            shell=False,
+            encoding="utf-8",
+        )
+        print(output)
+        shutil.move(output_name, f"generated/gha/{output_name}")
+        info(f"Built generated/gha/{output_name}")
+
+    # Back-compat: keep generated/gha/dagster-cloud.pex as a symlink to the
+    # x86_64 variant for one release, so any consumer reaching past the public
+    # action API (i.e. hardcoding the file path) keeps working on x86 runners.
+    legacy_path = "generated/gha/dagster-cloud.pex"
+    if os.path.lexists(legacy_path):
+        os.unlink(legacy_path)
+    os.symlink("dagster-cloud-x86_64.pex", legacy_path)
+    info(f"Linked {legacy_path} -> dagster-cloud-x86_64.pex")
 
 
 @app.command(help="Update dagster-cloud.pex")
diff --git a/src/Dockerfile b/src/Dockerfile
@@ -12,7 +12,7 @@ RUN /opt/python/cp311-cp311/bin/python -m pip install dagster-cloud-cli
 RUN /opt/python/cp312-cp312/bin/python -m pip install dagster-cloud-cli
 
 # Create virtual environment using PEX
-COPY generated/gha/dagster-cloud.pex /dagster-cloud.pex
+COPY generated/gha/dagster-cloud-x86_64.pex /dagster-cloud.pex
 RUN PEX_TOOLS=1 /opt/python/cp312-cp312/bin/python /dagster-cloud.pex venv /venv-dagster-cloud
 
 # Add all the relevant Python binaries to the PATH
diff --git a/src/Dockerfile.test-dagster-cloud-pex b/src/Dockerfile.test-dagster-cloud-pex
@@ -1,6 +1,6 @@
 # Used for testing the generated dagster-cloud pex
 FROM --platform=linux/amd64 python:3.10-slim
 
-COPY generated/gha/dagster-cloud.pex /dagster-cloud.pex
+COPY generated/gha/dagster-cloud-x86_64.pex /dagster-cloud.pex
 
 ENTRYPOINT ["/dagster-cloud.pex"]
diff --git a/src/deploy_pex.py b/src/deploy_pex.py
@@ -8,6 +8,7 @@
 # On anything else: forward args to `dagster-cloud --build-method=docker` to ensure they are built in a compatible environment
 
 import os
+import platform
 import re
 import subprocess
 import sys
@@ -16,7 +17,10 @@
 
 import yaml
 
-DAGSTER_CLOUD_PEX_PATH = Path(__file__).parent.parent / "generated/gha/dagster-cloud.pex"
+_ARCH = "aarch64" if platform.machine() == "aarch64" else "x86_64"
+DAGSTER_CLOUD_PEX_PATH = (
+    Path(__file__).parent.parent / f"generated/gha/dagster-cloud-{_ARCH}.pex"
+)
 UPDATE_COMMENT_SCRIPT_PATH = Path(__file__).parent / "create_or_update_comment.py"
 
 
