Skip to content

Commit 68780c0

Browse files
ryan-williamsryan-williams
committed
update demos
1 parent 4750558 commit 68780c0

7 files changed

Lines changed: 255 additions & 46 deletions

File tree

‎.github/workflows/demo-archs.yml‎

Lines changed: 46 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -1,70 +1,78 @@
1-
name: Demo – 2 GPU instances with different architectures
1+
name: Demo – x86 and ARM instances
22
on:
33
workflow_dispatch:
44
workflow_call: # Tested by `demos.yml`
55
permissions:
66
id-token: write # Required for AWS OIDC authentication
77
contents: read # Required for actions/checkout
88
jobs:
9-
# Launch EC2 runners for each instance type
10-
g4dn:
11-
name: Launch g4dn
9+
# Launch EC2 runners for each architecture
10+
x86:
11+
name: Launch x86
1212
uses: ./.github/workflows/runner.yml
1313
with:
14-
ec2_instance_type: g4dn.xlarge
14+
ec2_instance_type: t3.medium
15+
# Ubuntu 24.04 LTS x86_64 (default AMI)
1516
secrets: inherit
16-
g4ad:
17-
name: Launch g4ad
17+
arm:
18+
name: Launch ARM
1819
uses: ./.github/workflows/runner.yml
1920
with:
20-
ec2_instance_type: g4ad.xlarge
21+
ec2_instance_type: t4g.medium
22+
ec2_image_id: ami-0a07e880fa53d77f2 # Ubuntu 24.04 LTS arm64
2123
secrets: inherit
2224

2325
# Run jobs directly on launched instances
24-
test-g4dn:
25-
needs: g4dn
26-
if: needs.g4dn.outputs.id != ''
27-
name: Test g4dn
28-
runs-on: ${{ needs.g4dn.outputs.id }}
26+
test-x86:
27+
needs: x86
28+
if: needs.x86.outputs.id != ''
29+
name: Test x86
30+
runs-on: ${{ needs.x86.outputs.id }}
2931
steps:
30-
- name: GPU test on g4dn.xlarge
31-
run: nvidia-smi # Verify GPU is available
32-
test-g4ad:
33-
needs: g4ad
34-
if: needs.g4ad.outputs.id != ''
35-
name: Test g4ad
36-
runs-on: ${{ needs.g4ad.outputs.id }}
32+
- name: Architecture test on t3.medium
33+
run: |
34+
echo "Architecture: $(uname -m)"
35+
echo "Processor: $(lscpu | grep 'Architecture:' | awk '{print $2}')"
36+
echo "Instance type: $(curl -s http://169.254.169.254/latest/meta-data/instance-type)"
37+
test-arm:
38+
needs: arm
39+
if: needs.arm.outputs.id != ''
40+
name: Test ARM
41+
runs-on: ${{ needs.arm.outputs.id }}
3742
steps:
38-
- name: GPU test on g4ad.xlarge
43+
- name: Architecture test on t4g.medium
3944
run: |
40-
lspci | grep -i "vga\|display\|3d\|amd"
45+
echo "Architecture: $(uname -m)"
46+
echo "Processor: $(lscpu | grep 'Architecture:' | awk '{print $2}')"
47+
echo "Instance type: $(curl -s http://169.254.169.254/latest/meta-data/instance-type)"
4148
4249
# Use a `matrix` to run jobs on multiple instances
4350
test-matrix:
44-
needs: [g4dn, g4ad]
45-
if: always() && (needs.g4dn.outputs.id != '' || needs.g4ad.outputs.id != '')
46-
name: Test ${{ matrix.instance }} (matrix)
51+
needs: [x86, arm]
52+
if: always() && (needs.x86.outputs.id != '' || needs.arm.outputs.id != '')
53+
name: Test ${{ matrix.arch }} (matrix)
4754
continue-on-error: true
4855
strategy:
4956
matrix:
5057
include:
51-
- instance: g4dn
52-
runner: ${{ needs.g4dn.outputs.id }}
53-
- instance: g4ad
54-
runner: ${{ needs.g4ad.outputs.id }}
58+
- arch: x86
59+
runner: ${{ needs.x86.outputs.id }}
60+
- arch: arm
61+
runner: ${{ needs.arm.outputs.id }}
5562
fail-fast: false
5663
runs-on: ${{ matrix.runner }}
5764
steps:
58-
- name: Matrix GPU test on ${{ matrix.instance }}.xlarge
65+
- name: Matrix architecture test
5966
run: |
60-
echo "Running on ${{ matrix.instance }}.xlarge"
67+
echo "Running on ${{ matrix.arch }}"
68+
echo "Architecture: $(uname -m)"
6169
echo "Instance type: $(curl -s http://169.254.169.254/latest/meta-data/instance-type)"
6270
63-
# Check GPU based on instance type
64-
if [[ "${{ matrix.instance }}" == "g4dn" ]]; then
65-
echo "Testing NVIDIA GPU..."
66-
nvidia-smi
67-
elif [[ "${{ matrix.instance }}" == "g4ad" ]]; then
68-
echo "Testing AMD GPU..."
69-
lspci | grep -i "vga\|display\|3d\|amd"
71+
# Verify expected architecture
72+
if [[ "${{ matrix.arch }}" == "x86" ]]; then
73+
echo "Verifying x86_64 architecture..."
74+
[[ "$(uname -m)" == "x86_64" ]] && echo "✓ Confirmed x86_64" || exit 1
75+
elif [[ "${{ matrix.arch }}" == "arm" ]]; then
76+
echo "Verifying ARM architecture..."
77+
[[ "$(uname -m)" == "aarch64" ]] && echo "✓ Confirmed aarch64" || exit 1
7078
fi

‎.github/workflows/demo-multi-instance.yml‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
name: Demo – multiple instances for parallel jobs
2+
on:
3+
workflow_dispatch:
4+
inputs:
5+
instance_count:
6+
description: "Number of EC2 instances to create"
7+
required: false
8+
type: string
9+
default: "3"
10+
workflow_call: # Tested by `demos.yml`
11+
inputs:
12+
instance_count:
13+
required: false
14+
type: string
15+
default: "3"
16+
17+
permissions:
18+
id-token: write # Required for AWS OIDC authentication
19+
contents: read # Required for actions/checkout
20+
21+
jobs:
22+
ec2:
23+
name: Launch ${{ inputs.instance_count }} EC2 instances
24+
uses: ./.github/workflows/runner.yml
25+
secrets: inherit
26+
with:
27+
instance_count: ${{ inputs.instance_count }}
28+
29+
parallel-jobs:
30+
needs: ec2
31+
strategy:
32+
matrix:
33+
# Parse the JSON array of runner labels and use them in the matrix
34+
runner: ${{ fromJson(needs.ec2.outputs.instances) }}
35+
runs-on: ${{ matrix.runner }}
36+
name: Job on ${{ matrix.runner }}
37+
steps:
38+
- uses: actions/checkout@v4
39+
40+
- name: Instance info
41+
run: |
42+
echo "Running on instance with label: ${{ matrix.runner }}"
43+
echo "Hostname: $(hostname)"
44+
echo "Instance ID: $(curl -s http://169.254.169.254/latest/meta-data/instance-id)"
45+
echo "Instance type: $(curl -s http://169.254.169.254/latest/meta-data/instance-type)"
46+
echo "Region: $(curl -s http://169.254.169.254/latest/meta-data/placement/region)"
47+
48+
- name: Simulate workload
49+
run: |
50+
# Each instance runs independently
51+
DURATION=$((RANDOM % 10 + 5))
52+
echo "Simulating workload for ${DURATION} seconds..."
53+
sleep $DURATION
54+
echo "Workload complete!"
55+
56+
- name: Verify parallelism
57+
run: |
58+
echo "Completed at: $(date '+%Y-%m-%d %H:%M:%S')"
59+
echo "This job ran in parallel with other matrix jobs"

‎.github/workflows/demo-multi-job.yml‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
name: Demo – multiple instances for different job types
2+
on:
3+
workflow_dispatch:
4+
workflow_call: # Tested by `demos.yml`
5+
6+
permissions:
7+
id-token: write # Required for AWS OIDC authentication
8+
contents: read # Required for actions/checkout
9+
10+
jobs:
11+
ec2:
12+
name: Launch 2 EC2 instances
13+
uses: ./.github/workflows/runner.yml
14+
secrets: inherit
15+
with:
16+
instance_count: "2"
17+
18+
# First job type - runs on first instance
19+
build-job:
20+
needs: ec2
21+
runs-on: ${{ fromJson(needs.ec2.outputs.instances)[0] }}
22+
name: Build job
23+
steps:
24+
- uses: actions/checkout@v4
25+
26+
- name: Build task
27+
run: |
28+
echo "Running BUILD job on first instance"
29+
echo "Instance: $(curl -s http://169.254.169.254/latest/meta-data/instance-id)"
30+
echo "Simulating build process..."
31+
sleep 10
32+
echo "Build complete!"
33+
34+
- name: Create artifact
35+
run: |
36+
echo "Build output from $(hostname)" > build-output.txt
37+
echo "Built at $(date)" >> build-output.txt
38+
39+
- uses: actions/upload-artifact@v4
40+
with:
41+
name: build-output
42+
path: build-output.txt
43+
44+
# Second job type - runs on second instance
45+
test-job:
46+
needs: ec2
47+
runs-on: ${{ fromJson(needs.ec2.outputs.instances)[1] }}
48+
name: Test job
49+
steps:
50+
- uses: actions/checkout@v4
51+
52+
- name: Test task
53+
run: |
54+
echo "Running TEST job on second instance"
55+
echo "Instance: $(curl -s http://169.254.169.254/latest/meta-data/instance-id)"
56+
echo "Simulating test suite..."
57+
sleep 15
58+
echo "Tests passed!"
59+
60+
- name: Generate test report
61+
run: |
62+
echo "Test report from $(hostname)" > test-report.txt
63+
echo "Tests run at $(date)" >> test-report.txt
64+
echo "All tests: PASSED" >> test-report.txt
65+
66+
- uses: actions/upload-artifact@v4
67+
with:
68+
name: test-report
69+
path: test-report.txt
70+
71+
# Aggregation job that uses artifacts from both instances
72+
report:
73+
needs: [build-job, test-job]
74+
runs-on: ubuntu-latest
75+
steps:
76+
- name: Download all artifacts
77+
uses: actions/download-artifact@v4
78+
79+
- name: Generate final report
80+
run: |
81+
echo "=== Final Report ==="
82+
echo "Build output:"
83+
cat build-output/build-output.txt
84+
echo ""
85+
echo "Test report:"
86+
cat test-report/test-report.txt
87+
echo ""
88+
echo "Both jobs completed successfully using separate EC2 instances!"

‎.github/workflows/demos.yml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,3 +20,9 @@ jobs:
2020
demo-matrix-wide:
2121
uses: ./.github/workflows/demo-matrix-wide.yml
2222
secrets: inherit
23+
demo-multi-instance:
24+
uses: ./.github/workflows/demo-multi-instance.yml
25+
secrets: inherit
26+
demo-multi-job:
27+
uses: ./.github/workflows/demo-multi-job.yml
28+
secrets: inherit

‎.github/workflows/runner.yml‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,11 @@ on:
7575
description: "Additional userdata script to run on instance startup (before runner starts)"
7676
required: false
7777
type: string
78+
instance_count:
79+
description: "Number of EC2 instances to create (for parallel jobs)"
80+
required: false
81+
type: string
82+
default: "1"
7883
instance_name:
7984
description: "Name tag template for EC2 instances. Uses Python string.Template format with variables: $repo, $name (workflow filename stem), $workflow (full workflow name), $ref, $run_number"
8085
required: false
@@ -106,8 +111,14 @@ on:
106111
type: string
107112
outputs:
108113
id:
109-
description: "Instance ID for runs-on"
114+
description: "Instance ID for runs-on (single instance)"
110115
value: ${{ jobs.launch.outputs.id }}
116+
instances:
117+
description: "JSON array of runner labels (for multiple instances)"
118+
value: ${{ jobs.launch.outputs.instances }}
119+
mapping:
120+
description: "JSON object mapping instance IDs to runner labels"
121+
value: ${{ jobs.launch.outputs.mapping }}
111122

112123
permissions:
113124
id-token: write # Required for AWS OIDC
@@ -118,6 +129,8 @@ jobs:
118129
runs-on: ubuntu-latest
119130
outputs:
120131
id: ${{ steps.aws-start.outputs.label }}
132+
instances: ${{ steps.aws-start.outputs.instances }}
133+
mapping: ${{ steps.aws-start.outputs.mapping }}
121134
steps:
122135
- name: Check EC2_LAUNCH_ROLE configuration
123136
run: |
@@ -155,6 +168,7 @@ jobs:
155168
ec2_root_device_size: ${{ inputs.ec2_root_device_size }}
156169
ec2_security_group_id: ${{ inputs.ec2_security_group_id || vars.EC2_SECURITY_GROUP_ID }}
157170
ec2_userdata: ${{ inputs.ec2_userdata }}
171+
instance_count: ${{ inputs.instance_count }}
158172
instance_name: ${{ inputs.instance_name }}
159173
max_instance_lifetime: ${{ inputs.max_instance_lifetime || vars.MAX_INSTANCE_LIFETIME || '360' }}
160174
runner_initial_grace_period: ${{ inputs.runner_initial_grace_period || vars.RUNNER_INITIAL_GRACE_PERIOD || '180' }}

‎README.md‎

Lines changed: 36 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -82,14 +82,15 @@ Many of these fall back to corresponding `vars.*` (if not provided as `inputs`):
8282
- `action_ref` - ec2-gha Git ref to checkout (branch/tag/SHA); auto-detected if not specified
8383
- `cloudwatch_logs_group` - CloudWatch Logs group name for streaming logs (falls back to `vars.CLOUDWATCH_LOGS_GROUP`)
8484
- `ec2_home_dir` - Home directory (default: `/home/ubuntu`)
85-
- `ec2_image_id` - AMI ID (default: Deep Learning AMI)
85+
- `ec2_image_id` - AMI ID (default: Ubuntu 24.04 LTS)
8686
- `ec2_instance_profile` - IAM instance profile name for EC2 instances
8787
- Useful for on-instance debugging [via SSH][SSH access]
8888
- Required for [CloudWatch logging][cw]
8989
- Falls back to `vars.EC2_INSTANCE_PROFILE`
9090
- See [Appendix: IAM Role Setup](#iam-setup-appendix) for more details and sample setup code
91-
- `ec2_instance_type` - Instance type (default: `g4dn.xlarge`)
91+
- `ec2_instance_type` - Instance type (default: `t3.medium`)
9292
- `ec2_key_name` - EC2 key pair name (for [SSH access])
93+
- `instance_count` - Number of instances to create (default: 1, for parallel jobs)
9394
- `ec2_root_device_size` - Root device size in GB (default: 0 = use AMI default)
9495
- `ec2_security_group_id` - Security group ID (required for [SSH access], should expose inbound port 22)
9596
- `max_instance_lifetime` - Maximum instance lifetime in minutes before automatic shutdown (falls back to `vars.MAX_INSTANCE_LIFETIME`, default: 360 = 6 hours; generally should not be relevant, instances shut down within 1-2mins of jobs completing)
@@ -100,9 +101,11 @@ Many of these fall back to corresponding `vars.*` (if not provided as `inputs`):
100101

101102
## Outputs <a id="outputs"></a>
102103

103-
| Name | Description |
104-
|------|---------------------------------------------|
105-
| id | Value to pass to subsequent jobs' `runs-on` |
104+
| Name | Description |
105+
|------|--------------------------------------------------------------------------|
106+
| id | Single runner label for `runs-on` (when `instance_count=1`) |
107+
| instances | JSON array of runner labels (for use with matrix strategy) |
108+
| mapping | JSON object mapping instance IDs to labels (for debugging) |
106109

107110
## Technical Details <a id="technical"></a>
108111

@@ -115,7 +118,34 @@ This workflow creates EC2 instances with GitHub Actions runners that:
115118
- Use [GitHub's native runner hooks][hooks] for job tracking
116119
- Optionally support [SSH access] and [CloudWatch logging][cw] (for debugging)
117120

118-
### Multi-Job Workflows <a id="multi-job"></a>
121+
### Parallel Jobs (Multiple Instances) <a id="parallel"></a>
122+
123+
Create multiple EC2 instances for parallel execution using `instance_count`:
124+
125+
```yaml
126+
jobs:
127+
ec2:
128+
uses: Open-Athena/ec2-gha/.github/workflows/runner.yml@main
129+
secrets: inherit
130+
with:
131+
instance_count: "3" # Create 3 instances
132+
133+
parallel-jobs:
134+
needs: ec2
135+
strategy:
136+
matrix:
137+
runner: ${{ fromJson(needs.ec2.outputs.instances) }}
138+
runs-on: ${{ matrix.runner }}
139+
steps:
140+
- run: echo "Running on ${{ matrix.runner }}"
141+
```
142+
143+
Each instance gets a unique runner label and can execute jobs independently. This is useful for:
144+
- Matrix builds that need isolated environments
145+
- Parallel testing across different configurations
146+
- Distributed workloads
147+
148+
### Multi-Job Workflows (Sequential) <a id="multi-job"></a>
119149

120150
The runner supports multiple sequential jobs on the same instance, e.g.:
121151

0 commit comments

Comments
 (0)