chore(deps): bump the pip group across 5 directories with 10 updates #310
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: PR - PyTorch EC2 CUDA | |
| on: | |
| pull_request: | |
| branches: [main] | |
| types: [opened, reopened, synchronize] | |
| paths: | |
| - ".github/config/image/pytorch-*-ec2-cuda.yml" | |
| - ".github/workflows/pr-pytorch-ec2-cuda.yml" | |
| - "docker/pytorch/*/Dockerfile.cuda" | |
| - "docker/pytorch/*/cuda/**" | |
| - "docker/pytorch/*/versions-cuda.env" | |
| - "scripts/common/**" | |
| - "scripts/pytorch/**" | |
| - "scripts/telemetry/**" | |
| - ".github/scripts/efa/**" | |
| - ".github/workflows/reusable-efa-tests.yml" | |
| - "test/pytorch/**" | |
| - "test/efa/**" | |
| - "test/sanity/**" | |
| - "test/telemetry/**" | |
| - "!docs/**" | |
| permissions: | |
| contents: read | |
| pull-requests: read | |
| env: | |
| FORCE_COLOR: "1" | |
| LATEST_PYTORCH_VERSION: "2.11" | |
| jobs: | |
| # ============================================================ | |
| # Gate: permission check on base branch | |
| # ============================================================ | |
| gatekeeper: | |
| runs-on: ubuntu-latest | |
| concurrency: | |
| group: ${{ github.workflow }}-gate-${{ github.event.pull_request.number }} | |
| cancel-in-progress: true | |
| steps: | |
| - name: Checkout base branch (safe) | |
| uses: actions/checkout@v6 | |
| with: | |
| ref: ${{ github.event.pull_request.base.sha }} | |
| fetch-depth: 1 | |
| - name: Run permission gate (from base) | |
| uses: ./.github/actions/pr-permission-gate | |
| # ============================================================ | |
| # Detect all changed PyTorch versions + file changes | |
| # ============================================================ | |
| check-changes: | |
| needs: [gatekeeper] | |
| if: success() | |
| runs-on: ubuntu-latest | |
| concurrency: | |
| group: ${{ github.workflow }}-detect-${{ github.event.pull_request.number }} | |
| cancel-in-progress: true | |
| outputs: | |
| versions: ${{ steps.versions.outputs.versions }} | |
| configs: ${{ steps.versions.outputs.configs }} | |
| build-change: ${{ steps.changes.outputs.build-change }} | |
| sanity-test-change: ${{ steps.changes.outputs.sanity-test-change }} | |
| telemetry-test-change: ${{ steps.changes.outputs.telemetry-test-change }} | |
| efa-test-change: ${{ steps.changes.outputs.efa-test-change }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v6 | |
| - name: Setup python | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: "3.12" | |
| - name: Run pre-commit | |
| uses: pre-commit/action@v3.0.1 | |
| with: | |
| extra_args: --all-files | |
| - name: Install yq | |
| run: | | |
| if ! command -v yq &> /dev/null; then | |
| sudo wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 | |
| sudo chmod +x /usr/local/bin/yq | |
| fi | |
| - name: Detect PyTorch versions and build configs matrix | |
| id: versions | |
| run: | | |
| VERSIONS=$(git diff --name-only origin/main...HEAD \ | |
| | grep -oP '(?:docker/pytorch/|pytorch-)\K[0-9]+\.[0-9]+' \ | |
| | sort -u) | |
| if [ -z "$VERSIONS" ]; then | |
| VERSIONS="$LATEST_PYTORCH_VERSION" | |
| fi | |
| JSON=$(echo "$VERSIONS" | jq -R -s -c 'split("\n") | map(select(length > 0))') | |
| echo "versions=${JSON}" >> $GITHUB_OUTPUT | |
| echo "Detected versions: ${JSON}" | |
| # Build a configs matrix: each entry carries all metadata fields | |
| CONFIGS="[]" | |
| for V in $VERSIONS; do | |
| CONFIG_FILE=".github/config/image/pytorch-${V}-ec2-cuda.yml" | |
| if [ -f "$CONFIG_FILE" ]; then | |
| CONFIGS=$(echo "$CONFIGS" | jq -c \ | |
| --arg v "$V" \ | |
| --arg fw "$(yq '.common.framework' $CONFIG_FILE)" \ | |
| --arg fwv "$(yq '.common.framework_version' $CONFIG_FILE)" \ | |
| --arg py "$(yq '.common.python_version' $CONFIG_FILE)" \ | |
| --arg cuda "$(yq '.common.cuda_version' $CONFIG_FILE)" \ | |
| --arg os "$(yq '.common.os_version' $CONFIG_FILE)" \ | |
| --arg ct "$(yq '.common.job_type' $CONFIG_FILE)" \ | |
| --arg dt "$(yq '.common.device_type // "gpu"' $CONFIG_FILE)" \ | |
| --arg at "$(yq '.common.arch_type // "x86"' $CONFIG_FILE)" \ | |
| --arg contrib "$(yq '.common.contributor // "None"' $CONFIG_FILE)" \ | |
| --arg cust "$(yq '.common.customer_type // ""' $CONFIG_FILE)" \ | |
| --arg prod "$(yq '.common.prod_image' $CONFIG_FILE)" \ | |
| '. + [{"version": $v, "framework": $fw, "framework_version": $fwv, "python_version": $py, "cuda_version": $cuda, "os_version": $os, "container_type": $ct, "device_type": $dt, "arch_type": $at, "contributor": $contrib, "customer_type": $cust, "prod_image": $prod}]') | |
| fi | |
| done | |
| echo "configs=${CONFIGS}" >> $GITHUB_OUTPUT | |
| echo "Configs matrix: ${CONFIGS}" | |
| - name: Detect file changes | |
| id: changes | |
| uses: dorny/paths-filter@v4 | |
| with: | |
| filters: | | |
| build-change: | |
| - ".github/config/image/pytorch-*-ec2-cuda.yml" | |
| - "docker/pytorch/*/Dockerfile.cuda" | |
| - "docker/pytorch/*/cuda/**" | |
| - "scripts/common/setup_oss_compliance.sh" | |
| - "scripts/pytorch/*" | |
| - "scripts/telemetry/bash_telemetry.sh.template" | |
| sanity-test-change: | |
| - "test/sanity/**" | |
| telemetry-test-change: | |
| - "test/telemetry/**" | |
| efa-test-change: | |
| - "test/efa/**" | |
| - ".github/scripts/efa/**" | |
| - ".github/workflows/reusable-efa-tests.yml" | |
| # ============================================================ | |
| # Build images (matrix over detected versions) | |
| # ============================================================ | |
| build-images: | |
| needs: [check-changes] | |
| if: needs.check-changes.outputs.build-change == 'true' | |
| runs-on: | |
| - codebuild-runner-${{ github.run_id }}-${{ github.run_attempt }} | |
| fleet:x86-build-runner | |
| buildspec-override:true | |
| strategy: | |
| matrix: | |
| include: ${{ fromJson(needs.check-changes.outputs.configs) }} | |
| fail-fast: false | |
| concurrency: | |
| group: ${{ github.workflow }}-build-${{ matrix.version }}-${{ github.event.pull_request.number }} | |
| cancel-in-progress: true | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v6 | |
| - name: Setup buildkitd | |
| run: .github/scripts/buildkitd.sh | |
| - name: ECR login | |
| uses: ./.github/actions/ecr-authenticate | |
| with: | |
| aws-account-id: ${{ vars.CI_AWS_ACCOUNT_ID }} | |
| aws-region: ${{ vars.AWS_REGION }} | |
| - name: Fetch cached wheels | |
| run: | | |
| VERSION="${{ matrix.version }}" | |
| source docker/pytorch/${VERSION}/versions-cuda.env | |
| mkdir -p docker/pytorch/${VERSION}/wheels | |
| bash scripts/pytorch/fetch_cached_wheels.sh \ | |
| docker/pytorch/${VERSION}/wheels \ | |
| "${{ vars.WHEEL_CACHE_BUCKET }}" \ | |
| "${CUDA_VERSION}" "${TORCH_VERSION}" "${PYTHON_VERSION}" \ | |
| "flash-attn:${FLASH_ATTN_VERSION}" \ | |
| "transformer-engine-torch:${TRANSFORMER_ENGINE_VERSION}" | |
| continue-on-error: true | |
| - name: Build runtime image | |
| id: build-runtime | |
| run: | | |
| VERSION="${{ matrix.version }}" | |
| source docker/pytorch/${VERSION}/versions-cuda.env | |
| CI_IMAGE_URI="${{ vars.CI_AWS_ACCOUNT_ID }}.dkr.ecr.${{ vars.AWS_REGION }}.amazonaws.com/ci:pytorch-runtime-${{ matrix.version }}-pr-${{ github.event.pull_request.number }}" | |
| docker buildx build --progress plain \ | |
| --build-arg DLC_PYTORCH_VERSION=${{ matrix.version }} \ | |
| --build-arg FRAMEWORK=${{ matrix.framework }} \ | |
| --build-arg CUDA_VERSION=${CUDA_VERSION} \ | |
| --build-arg PYTHON_VERSION=${PYTHON_VERSION} \ | |
| --build-arg TORCH_VERSION=${TORCH_VERSION} \ | |
| --build-arg DLC_MAJOR_VERSION=${DLC_MAJOR_VERSION} \ | |
| --build-arg DLC_MINOR_VERSION=${DLC_MINOR_VERSION} \ | |
| --build-arg FLASH_ATTN_VERSION=${FLASH_ATTN_VERSION} \ | |
| --build-arg TRANSFORMER_ENGINE_VERSION=${TRANSFORMER_ENGINE_VERSION} \ | |
| --build-arg GDRCOPY_VERSION=${GDRCOPY_VERSION} \ | |
| --build-arg EFA_VERSION=${EFA_VERSION} \ | |
| --build-arg MAX_JOBS=${MAX_JOBS} \ | |
| --cache-to=type=inline \ | |
| --cache-from=type=registry,ref=${CI_IMAGE_URI} \ | |
| --tag ${CI_IMAGE_URI} \ | |
| --push \ | |
| --target runtime \ | |
| -f docker/pytorch/${VERSION}/Dockerfile.cuda . | |
| echo "image-uri=${CI_IMAGE_URI}" >> $GITHUB_OUTPUT | |
| - name: Upload built wheels to cache | |
| run: | | |
| VERSION="${{ matrix.version }}" | |
| source docker/pytorch/${VERSION}/versions-cuda.env | |
| bash scripts/pytorch/upload_cached_wheels.sh \ | |
| "${{ vars.WHEEL_CACHE_BUCKET }}" \ | |
| "${CUDA_VERSION}" "${TORCH_VERSION}" "${PYTHON_VERSION}" \ | |
| "${{ steps.build-runtime.outputs.image-uri }}" \ | |
| "docker/pytorch/${VERSION}/Dockerfile.cuda" \ | |
| "flash-attn:${FLASH_ATTN_VERSION}" \ | |
| "transformer-engine-torch:${TRANSFORMER_ENGINE_VERSION}" | |
| continue-on-error: true | |
| # ============================================================ | |
| # Unit tests (CPU-only, no GPU needed) | |
| # ============================================================ | |
| unit-test: | |
| needs: [check-changes, build-images] | |
| if: success() | |
| strategy: | |
| matrix: | |
| include: ${{ fromJson(needs.check-changes.outputs.configs) }} | |
| fail-fast: false | |
| runs-on: | |
| - codebuild-runner-${{ github.run_id }}-${{ github.run_attempt }} | |
| fleet:default-runner | |
| buildspec-override:true | |
| concurrency: | |
| group: ${{ github.workflow }}-unit-${{ matrix.version }}-${{ github.event.pull_request.number }} | |
| cancel-in-progress: true | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v6 | |
| - name: ECR login | |
| uses: ./.github/actions/ecr-authenticate | |
| with: | |
| aws-account-id: ${{ vars.CI_AWS_ACCOUNT_ID }} | |
| aws-region: ${{ vars.AWS_REGION }} | |
| - name: Run unit tests | |
| run: | | |
| VERSION="${{ matrix.version }}" | |
| IMAGE="${{ vars.CI_AWS_ACCOUNT_ID }}.dkr.ecr.${{ vars.AWS_REGION }}.amazonaws.com/ci:pytorch-runtime-${{ matrix.version }}-pr-${{ github.event.pull_request.number }}" | |
| docker pull ${IMAGE} | |
| CONTAINER_ID=$(docker run -d --rm --entrypoint /bin/bash \ | |
| -e DLC_WORKDIR=/workdir -e DLC_PYTORCH_VERSION=${VERSION} \ | |
| -v $(pwd):/workdir --workdir /workdir \ | |
| ${IMAGE} -c 'sleep infinity') | |
| docker exec ${CONTAINER_ID} pip install pytest -q | |
| docker exec ${CONTAINER_ID} pytest /workdir/test/pytorch/unit/ -v | |
| docker kill ${CONTAINER_ID} | |
| # ============================================================ | |
| # Sanity tests (labels, filesystem, OSS compliance) | |
| # ============================================================ | |
| sanity-test: | |
| needs: [check-changes, build-images] | |
| if: | | |
| always() && !failure() && !cancelled() && | |
| (needs.check-changes.outputs.build-change == 'true' || needs.check-changes.outputs.sanity-test-change == 'true') | |
| strategy: | |
| matrix: | |
| include: ${{ fromJson(needs.check-changes.outputs.configs) }} | |
| fail-fast: false | |
| uses: ./.github/workflows/reusable-sanity-tests.yml | |
| with: | |
| image-uri: ${{ needs.build-images.result == 'success' && format('{0}.dkr.ecr.{1}.amazonaws.com/ci:pytorch-runtime-{2}-pr-{3}', vars.CI_AWS_ACCOUNT_ID, vars.AWS_REGION, matrix.version, github.event.pull_request.number) || format('{0}.dkr.ecr.{1}.amazonaws.com/{2}', vars.PROD_AWS_ACCOUNT_ID, vars.AWS_REGION, matrix.prod_image) }} | |
| aws-account-id: ${{ needs.build-images.result == 'success' && vars.CI_AWS_ACCOUNT_ID || vars.PROD_AWS_ACCOUNT_ID }} | |
| aws-region: ${{ vars.AWS_REGION }} | |
| framework: ${{ matrix.framework }} | |
| framework-version: ${{ matrix.framework_version }} | |
| python-version: ${{ matrix.python_version }} | |
| cuda-version: ${{ matrix.cuda_version }} | |
| os-version: ${{ matrix.os_version }} | |
| customer-type: ${{ matrix.customer_type }} | |
| arch-type: ${{ matrix.arch_type }} | |
| device-type: ${{ matrix.device_type }} | |
| contributor: ${{ matrix.contributor }} | |
| container-type: ${{ matrix.container_type }} | |
| # ============================================================ | |
| # Security tests (ECR scan, CVE allowlist) | |
| # ============================================================ | |
| security-test: | |
| needs: [check-changes, build-images] | |
| if: success() | |
| strategy: | |
| matrix: | |
| include: ${{ fromJson(needs.check-changes.outputs.configs) }} | |
| fail-fast: false | |
| uses: ./.github/workflows/reusable-security-tests.yml | |
| with: | |
| image-uri: ${{ needs.build-images.result == 'success' && format('{0}.dkr.ecr.{1}.amazonaws.com/ci:pytorch-runtime-{2}-pr-{3}', vars.CI_AWS_ACCOUNT_ID, vars.AWS_REGION, matrix.version, github.event.pull_request.number) || format('{0}.dkr.ecr.{1}.amazonaws.com/{2}', vars.PROD_AWS_ACCOUNT_ID, vars.AWS_REGION, matrix.prod_image) }} | |
| aws-account-id: ${{ needs.build-images.result == 'success' && vars.CI_AWS_ACCOUNT_ID || vars.PROD_AWS_ACCOUNT_ID }} | |
| aws-region: ${{ vars.AWS_REGION }} | |
| framework: ${{ matrix.framework }} | |
| framework-version: ${{ matrix.framework_version }} | |
| # ============================================================ | |
| # Telemetry tests (opt-out, environment variables) | |
| # ============================================================ | |
| telemetry-test: | |
| needs: [check-changes, build-images] | |
| if: | | |
| always() && !failure() && !cancelled() && | |
| (needs.check-changes.outputs.build-change == 'true' || needs.check-changes.outputs.telemetry-test-change == 'true') | |
| strategy: | |
| matrix: | |
| include: ${{ fromJson(needs.check-changes.outputs.configs) }} | |
| fail-fast: false | |
| uses: ./.github/workflows/reusable-telemetry-tests.yml | |
| with: | |
| image-uri: ${{ needs.build-images.result == 'success' && format('{0}.dkr.ecr.{1}.amazonaws.com/ci:pytorch-runtime-{2}-pr-{3}', vars.CI_AWS_ACCOUNT_ID, vars.AWS_REGION, matrix.version, github.event.pull_request.number) || format('{0}.dkr.ecr.{1}.amazonaws.com/{2}', vars.PROD_AWS_ACCOUNT_ID, vars.AWS_REGION, matrix.prod_image) }} | |
| aws-account-id: ${{ needs.build-images.result == 'success' && vars.CI_AWS_ACCOUNT_ID || vars.PROD_AWS_ACCOUNT_ID }} | |
| aws-region: ${{ vars.AWS_REGION }} | |
| framework: ${{ matrix.framework }} | |
| framework-version: ${{ matrix.framework_version }} | |
| container-type: ${{ matrix.container_type }} | |
| # ============================================================ | |
| # Single-GPU tests | |
| # ============================================================ | |
| single-gpu-test: | |
| needs: [check-changes, build-images, unit-test, sanity-test, security-test] | |
| if: success() | |
| strategy: | |
| matrix: | |
| version: ${{ fromJson(needs.check-changes.outputs.versions) }} | |
| fail-fast: false | |
| runs-on: | |
| - codebuild-runner-${{ github.run_id }}-${{ github.run_attempt }} | |
| fleet:x86-g6xl-runner | |
| buildspec-override:true | |
| concurrency: | |
| group: ${{ github.workflow }}-single-gpu-${{ matrix.version }}-${{ github.event.pull_request.number }} | |
| cancel-in-progress: true | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v6 | |
| - name: ECR login | |
| uses: ./.github/actions/ecr-authenticate | |
| with: | |
| aws-account-id: ${{ vars.CI_AWS_ACCOUNT_ID }} | |
| aws-region: ${{ vars.AWS_REGION }} | |
| - name: Run single-GPU tests | |
| run: | | |
| IMAGE="${{ vars.CI_AWS_ACCOUNT_ID }}.dkr.ecr.${{ vars.AWS_REGION }}.amazonaws.com/ci:pytorch-runtime-${{ matrix.version }}-pr-${{ github.event.pull_request.number }}" | |
| docker pull ${IMAGE} | |
| CONTAINER_ID=$(docker run -d --rm --gpus all --shm-size=2g \ | |
| --entrypoint /bin/bash \ | |
| -e DLC_WORKDIR=/workdir \ | |
| -v $(pwd):/workdir --workdir /workdir \ | |
| ${IMAGE} -c 'sleep infinity') | |
| docker exec ${CONTAINER_ID} pip install pytest -q | |
| docker exec ${CONTAINER_ID} pytest /workdir/test/pytorch/single_gpu/ -v | |
| docker kill ${CONTAINER_ID} | |
| # ============================================================ | |
| # EFA integration test (2x p4d.24xlarge, NCCL over EFA) | |
| # ============================================================ | |
| efa-test: | |
| needs: [check-changes, build-images, unit-test, sanity-test, security-test] | |
| if: | | |
| always() && !failure() && !cancelled() && | |
| (needs.check-changes.outputs.build-change == 'true' || needs.check-changes.outputs.efa-test-change == 'true') && | |
| (needs.build-images.result == 'success' || needs.build-images.result == 'skipped') | |
| strategy: | |
| matrix: | |
| include: ${{ fromJson(needs.check-changes.outputs.configs) }} | |
| fail-fast: false | |
| concurrency: | |
| group: ${{ github.workflow }}-efa-${{ matrix.version }}-${{ github.event.pull_request.number }} | |
| cancel-in-progress: true | |
| uses: ./.github/workflows/reusable-efa-tests.yml | |
| with: | |
| image-uri: ${{ needs.build-images.result == 'success' && format('{0}.dkr.ecr.{1}.amazonaws.com/ci:pytorch-runtime-{2}-pr-{3}', vars.CI_AWS_ACCOUNT_ID, vars.AWS_REGION, matrix.version, github.event.pull_request.number) || format('{0}.dkr.ecr.{1}.amazonaws.com/{2}', vars.PROD_AWS_ACCOUNT_ID, vars.AWS_REGION, matrix.prod_image) }} | |
| aws-account-id: ${{ needs.build-images.result == 'success' && vars.CI_AWS_ACCOUNT_ID || vars.PROD_AWS_ACCOUNT_ID }} | |
| aws-region: ${{ vars.AWS_REGION }} | |
| # ============================================================ | |
| # Multi-GPU tests (need >=2 GPUs) | |
| # TODO: Re-enable when GPU capacity is available | |
| # ============================================================ | |
| # multi-gpu-test: | |
| # needs: [build-images, single-gpu-test] | |
| # if: success() | |
| # runs-on: | |
| # - codebuild-runner-${{ github.run_id }}-${{ github.run_attempt }} | |
| # fleet:x86-g6e12xl-runner | |
| # buildspec-override:true | |
| # concurrency: | |
| # group: ${{ github.workflow }}-multi-gpu-${{ github.event.pull_request.number }} | |
| # cancel-in-progress: true | |
| # steps: | |
| # - name: Checkout code | |
| # uses: actions/checkout@v6 | |
| # | |
| # - name: ECR login | |
| # uses: ./.github/actions/ecr-authenticate | |
| # with: | |
| # aws-account-id: ${{ vars.CI_AWS_ACCOUNT_ID }} | |
| # aws-region: ${{ vars.AWS_REGION }} | |
| # | |
| # - name: Run multi-GPU tests | |
| # run: | | |
| # IMAGE="${{ needs.build-images.outputs.runtime-image-uri }}" | |
| # docker pull ${IMAGE} | |
| # CONTAINER_ID=$(docker run -d --rm --gpus all --shm-size=2g \ | |
| # --entrypoint /bin/bash \ | |
| # -e DLC_WORKDIR=/workdir \ | |
| # -v $(pwd):/workdir --workdir /workdir \ | |
| # ${IMAGE} -c 'sleep infinity') | |
| # docker exec ${CONTAINER_ID} pip install pytest -q | |
| # | |
| # docker exec ${CONTAINER_ID} torchrun --nproc_per_node=2 --master_port=29500 \ | |
| # /workdir/test/pytorch/multi_gpu/test_ddp.py | |
| # docker exec ${CONTAINER_ID} torchrun --nproc_per_node=2 --master_port=29501 \ | |
| # /workdir/test/pytorch/multi_gpu/test_fsdp.py | |
| # docker exec ${CONTAINER_ID} torchrun --nproc_per_node=2 --master_port=29502 \ | |
| # /workdir/test/pytorch/multi_gpu/test_deepspeed.py | |
| # | |
| # docker kill ${CONTAINER_ID} | |
| # ============================================================ | |
| # Multi-node tests (need 2+ containers on Docker network) | |
| # TODO: Re-enable when GPU capacity is available | |
| # ============================================================ | |
| # multi-node-test: | |
| # needs: [build-images, multi-gpu-test] | |
| # if: success() | |
| # runs-on: | |
| # - codebuild-runner-${{ github.run_id }}-${{ github.run_attempt }} | |
| # fleet:x86-g6e12xl-runner | |
| # buildspec-override:true | |
| # concurrency: | |
| # group: ${{ github.workflow }}-multi-node-${{ github.event.pull_request.number }} | |
| # cancel-in-progress: true | |
| # steps: | |
| # - name: Checkout code | |
| # uses: actions/checkout@v6 | |
| # | |
| # - name: ECR login | |
| # uses: ./.github/actions/ecr-authenticate | |
| # with: | |
| # aws-account-id: ${{ vars.CI_AWS_ACCOUNT_ID }} | |
| # aws-region: ${{ vars.AWS_REGION }} | |
| # | |
| # - name: Run multi-node tests | |
| # run: | | |
| # IMAGE="${{ needs.build-images.outputs.runtime-image-uri }}" | |
| # docker pull ${IMAGE} | |
| # docker network create testnet | |
| # | |
| # NODE0=$(docker run -d --rm --gpus '"device=0"' --shm-size=2g \ | |
| # --network=testnet --hostname=node0 --name=node0 \ | |
| # --entrypoint /bin/bash \ | |
| # -v $(pwd):/workdir --workdir /workdir \ | |
| # ${IMAGE} -c 'sleep infinity') | |
| # | |
| # NODE1=$(docker run -d --rm --gpus '"device=1"' --shm-size=2g \ | |
| # --network=testnet --hostname=node1 --name=node1 \ | |
| # --entrypoint /bin/bash \ | |
| # -v $(pwd):/workdir --workdir /workdir \ | |
| # ${IMAGE} -c 'sleep infinity') | |
| # | |
| # # DDP multi-node | |
| # docker exec node0 torchrun --nnodes=2 --nproc_per_node=1 --node_rank=0 \ | |
| # --master_addr=node0 --master_port=29400 \ | |
| # /workdir/test/pytorch/multi_node/test_multinode_ddp.py & | |
| # docker exec node1 torchrun --nnodes=2 --nproc_per_node=1 --node_rank=1 \ | |
| # --master_addr=node0 --master_port=29400 \ | |
| # /workdir/test/pytorch/multi_node/test_multinode_ddp.py & | |
| # wait | |
| # | |
| # # NCCL all_reduce | |
| # docker exec node0 torchrun --nnodes=2 --nproc_per_node=1 --node_rank=0 \ | |
| # --master_addr=node0 --master_port=29401 \ | |
| # /workdir/test/pytorch/multi_node/test_nccl_efa.py & | |
| # docker exec node1 torchrun --nnodes=2 --nproc_per_node=1 --node_rank=1 \ | |
| # --master_addr=node0 --master_port=29401 \ | |
| # /workdir/test/pytorch/multi_node/test_nccl_efa.py & | |
| # wait | |
| # | |
| # docker kill node0 node1 | |
| # docker network rm testnet | |
| # ============================================================ | |
| # Integration tests (EFA, SageMaker, EKS) | |
| # TODO: Enable when integration test infrastructure is available | |
| # See test/pytorch/integration/TODO.md for planned tests | |
| # ============================================================ | |
| # integration-test: | |
| # needs: [multi-node-test] | |
| # ... |