Skip to content

Commit 56534e7

Browse files
viktor-kurchenkoviktor-kurchenko
committed
infra,ci: arm64 runner configs and action
The commit implements: - infra configurations for the self-hosted ARM64 GitHub runner based on AWS EC2 instance - reusable GitHUb action to manage ARM64 self-hosted runner Signed-off-by: viktor-kurchenko <[email protected]>
1 parent 767a159 commit 56534e7

File tree

7 files changed

+507
-0
lines changed

7 files changed

+507
-0
lines changed

‎.github/actions/arm64-runner/action.yaml

Lines changed: 168 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,168 @@
1+
name: Manage ARM64 runners
2+
description: Manage ARM64 runners
3+
inputs:
4+
state-bucket:
5+
required: true
6+
description: "AWS S3 bucket name for the Terraform state"
7+
state-region:
8+
required: true
9+
description: "AWS region where state bucket has been created"
10+
role-arn:
11+
required: true
12+
description: "AWS IAM role ARN"
13+
region:
14+
required: true
15+
description: "AWS region"
16+
zone:
17+
required: true
18+
description: "AWS availability zone"
19+
infra-dir:
20+
required: true
21+
description: "Directory with infra config"
22+
action:
23+
required: true
24+
description: "Terraform action: plan, apply, destroy"
25+
ec2-type:
26+
required: true
27+
description: "EC2 instance type"
28+
ec2-ami:
29+
required: true
30+
description: "EC2 AMI type"
31+
label:
32+
required: true
33+
description: "Runners label"
34+
gh-org:
35+
required: true
36+
description: "GitHub organization"
37+
gh-app-id:
38+
required: true
39+
description: "GitHub APP ID"
40+
gh-app-install-id:
41+
required: true
42+
description: "GitHub APP install ID"
43+
gh-app-pem:
44+
required: true
45+
description: "GitHub APP private key"
46+
gh-runners-group:
47+
required: true
48+
description: "GitHub self-hosted runners group"
49+
gh-runners-count:
50+
required: true
51+
description: "GitHub virtual runners count"
52+
ssh-private-key:
53+
required: true
54+
description: "SSH private key for the runner access"
55+
ssh-public-key:
56+
required: true
57+
description: "SSH public key for the runner access"
58+
59+
runs:
60+
using: composite
61+
steps:
62+
- name: Install Terraform
63+
uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd # 3.1.2
64+
with:
65+
terraform_version: "1.10.3"
66+
67+
- name: Set up AWS CLI credentials
68+
uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
69+
with:
70+
role-to-assume: ${{ inputs.role-arn }}
71+
aws-region: ${{ inputs.region }}
72+
73+
- name: Lookup EC2 instance FQDN
74+
if: ${{ inputs.action == 'destroy' }}
75+
id: ec2-destroy
76+
shell: bash
77+
run: |
78+
fqdn="$(aws ec2 describe-instances --region ${{ inputs.region }} | \
79+
jq -r '.Reservations[].Instances[] | select(.State.Name == "running") | select(.Tags != null) | select(.Tags[].Value|test("^${{ inputs.label }}$")) | .PublicDnsName')"
80+
echo fqdn=$fqdn >> $GITHUB_OUTPUT
81+
82+
- name: Issue GH token
83+
if: ${{ inputs.action == 'destroy' }}
84+
id: gh
85+
shell: bash
86+
working-directory: ${{ inputs.infra-dir }}
87+
run: |
88+
export GITHUB_ORG="${{ inputs.gh-org }}"
89+
export GITHUB_APP_ID="${{ inputs.gh-app-id }}"
90+
export GITHUB_APP_INSTALL_ID="${{ inputs.gh-app-install-id }}"
91+
export GITHUB_APP_PEM="${{ inputs.gh-app-pem }}"
92+
token=$(./token.sh)
93+
echo token=$token >> $GITHUB_OUTPUT
94+
95+
- name: Delete GH runners
96+
if: ${{ inputs.action == 'destroy' }}
97+
uses: appleboy/ssh-action@8faa84277b88b6cd1455986f459aa66cf72bc8a3 # v1.2.1
98+
with:
99+
host: ${{ steps.ec2-destroy.outputs.fqdn }}
100+
username: ubuntu
101+
key: ${{ inputs.ssh-private-key }}
102+
script: |
103+
echo "GH runners list:"
104+
/multi-runners/mr.bash list
105+
106+
for (( i=1; i<=${{ inputs.gh-runners-count }}; i++ ))
107+
do
108+
echo "deleting runner-$i ..."
109+
/multi-runners/mr.bash del --user runner-$i --token ${{ steps.gh.outputs.token }} || true
110+
done
111+
112+
- name: Set up Terraform variables
113+
working-directory: ${{ inputs.infra-dir }}
114+
shell: bash
115+
run: |
116+
cat > terraform.tfvars << EOF
117+
ssh_key_pair="${{ github.repository }}/${{ inputs.label }}"
118+
ssh_public_key="${{ inputs.ssh-public-key }}"
119+
owner="${{ github.repository }}"
120+
region="${{ inputs.region }}"
121+
zone="${{ inputs.zone }}"
122+
gh_app_id="${{ inputs.gh-app-id }}"
123+
gh_app_install_id="${{ inputs.gh-app-install-id }}"
124+
gh_org="${{ inputs.gh-org }}"
125+
gh_group="${{ inputs.gh-runners-group }}"
126+
gh_runners_count="${{ inputs.gh-runners-count }}"
127+
gh_label="${{ inputs.label }}"
128+
ec2_type="${{ inputs.ec2-type }}"
129+
ec2_ami="${{ inputs.ec2-ami }}"
130+
gh_app_pem=<<EOT
131+
${{ inputs.gh-app-pem }}
132+
EOT
133+
EOF
134+
135+
- name: ${{ inputs.action }} runner
136+
working-directory: ${{ inputs.infra-dir }}
137+
shell: bash
138+
run: |
139+
make ${{ inputs.action }} STATE_REGION=${{ inputs.state-region }} \
140+
STATE_BUCKET=${{ inputs.state-bucket }} \
141+
STATE_FILE=${{ inputs.label }} \
142+
AUTO_APPROVE=true
143+
144+
- name: Lookup EC2 instance FQDN
145+
if: ${{ inputs.action == 'apply' }}
146+
id: ec2-apply
147+
shell: bash
148+
run: |
149+
for (( i=1; i<=60; i++ ))
150+
do
151+
sleep 10
152+
id_fqdn="$(aws ec2 describe-instances --region ${{ inputs.region }} | \
153+
jq -r '.Reservations[].Instances[] | select(.Tags != null) | select(.Tags[].Value|test("^${{ inputs.label }}$")) | .InstanceId + "," + .PublicDnsName')" || true
154+
id=$(echo "$id_fqdn" | cut -d "," -f 1)
155+
fqdn=$(echo "$id_fqdn" | cut -d "," -f 2)
156+
ready_count="$(aws ec2 describe-instance-status --no-cli-pager --instance-ids $id --region ${{ inputs.region }} | \
157+
jq '.InstanceStatuses[] | select(.InstanceStatus.Details[].Status == "passed") | .InstanceId ' | wc -l)" || true
158+
if [[ $ready_count -eq 1 ]]
159+
then
160+
echo "EC2 instance is ready now [id: $id, fqdn: $fqdn]"
161+
echo fqdn=$fqdn >> $GITHUB_OUTPUT
162+
exit 0
163+
fi
164+
echo "EC2 instance is not ready yet ..."
165+
done
166+
167+
echo "EC2 instance not ready!"
168+
exit 1

‎infra/arm64-runner/Makefile

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
SHELL := /bin/bash
2+
3+
.PHONY: plan apply destroy init
4+
5+
init:
6+
terraform init --upgrade \
7+
-backend-config=region=$(STATE_REGION) \
8+
-backend-config=bucket=$(STATE_BUCKET) \
9+
-backend-config=key=$(STATE_FILE)
10+
11+
plan: init
12+
terraform plan
13+
14+
apply: AUTO_APPROVE ?= false
15+
apply: init
16+
terraform apply --auto-approve=$(AUTO_APPROVE)
17+
18+
destroy: AUTO_APPROVE ?= false
19+
destroy: init
20+
terraform destroy --auto-approve=$(AUTO_APPROVE)

‎infra/arm64-runner/main.tf

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
# VPC setup
2+
resource "aws_vpc" "vpc" {
3+
cidr_block = "10.0.0.0/16"
4+
enable_dns_hostnames = true
5+
enable_dns_support = true
6+
7+
tags = {
8+
Name = "${var.owner}/${var.gh_label}-runner"
9+
}
10+
}
11+
12+
resource "aws_subnet" "subnet" {
13+
vpc_id = aws_vpc.vpc.id
14+
cidr_block = "10.0.1.0/24"
15+
availability_zone = var.zone
16+
}
17+
18+
resource "aws_security_group" "security_group" {
19+
name = "allow-all"
20+
21+
vpc_id = aws_vpc.vpc.id
22+
23+
ingress {
24+
cidr_blocks = [
25+
"0.0.0.0/0"
26+
]
27+
from_port = 22
28+
to_port = 22
29+
protocol = "tcp"
30+
}
31+
32+
egress {
33+
from_port = 0
34+
to_port = 0
35+
protocol = -1
36+
cidr_blocks = ["0.0.0.0/0"]
37+
}
38+
}
39+
40+
resource "aws_internet_gateway" "gateway" {
41+
vpc_id = aws_vpc.vpc.id
42+
}
43+
44+
resource "aws_route_table" "route_table" {
45+
vpc_id = aws_vpc.vpc.id
46+
47+
48+
route {
49+
cidr_block = "0.0.0.0/0"
50+
gateway_id = aws_internet_gateway.gateway.id
51+
}
52+
}
53+
54+
resource "aws_route_table_association" "route_table_association" {
55+
subnet_id = aws_subnet.subnet.id
56+
route_table_id = aws_route_table.route_table.id
57+
}
58+
59+
# SSH key
60+
resource "aws_key_pair" "key_pair" {
61+
key_name = var.ssh_key_pair
62+
public_key = var.ssh_public_key
63+
}
64+
65+
# EC2 instance
66+
resource "aws_instance" "runner" {
67+
instance_type = var.ec2_type
68+
ami = var.ec2_ami
69+
key_name = var.ssh_key_pair
70+
associate_public_ip_address = true
71+
subnet_id = aws_subnet.subnet.id
72+
vpc_security_group_ids = ["${aws_security_group.security_group.id}"]
73+
74+
root_block_device {
75+
volume_size = 256
76+
volume_type = "gp3"
77+
}
78+
79+
user_data = replace(replace(replace(replace(replace(replace(replace(file("./setup.sh"),
80+
"{GITHUB_APP_ID}", var.gh_app_id),
81+
"{GITHUB_APP_INSTALL_ID}", var.gh_app_install_id),
82+
"{GITHUB_APP_PEM}", var.gh_app_pem),
83+
"{GITHUB_ORG}", var.gh_org),
84+
"{GITHUB_GROUP}", var.gh_group),
85+
"{GITHUB_LABELS}", var.gh_label),
86+
"{GITHUB_RUNNERS_COUNT}", var.gh_runners_count)
87+
88+
lifecycle {
89+
ignore_changes = [user_data]
90+
}
91+
92+
tags = {
93+
Name = "${var.owner}/${var.gh_label}"
94+
Label = "${var.gh_label}"
95+
}
96+
}
97+

‎infra/arm64-runner/providers.tf

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
terraform {
2+
required_providers {
3+
aws = {
4+
source = "hashicorp/aws"
5+
version = "~> 5.55.0"
6+
}
7+
}
8+
9+
backend "s3" {
10+
encrypt = true
11+
}
12+
}
13+
14+
provider "aws" {
15+
region = var.region
16+
17+
default_tags {
18+
tags = {
19+
owner = var.owner
20+
}
21+
}
22+
}

‎infra/arm64-runner/setup.sh

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
#!/bin/bash
2+
3+
set -e
4+
5+
GITHUB_API_SERVER="api.github.com"
6+
# the following lines will be replaced by Terraform
7+
GITHUB_APP_ID="{GITHUB_APP_ID}"
8+
GITHUB_APP_INSTALL_ID="{GITHUB_APP_INSTALL_ID}"
9+
GITHUB_APP_PEM="{GITHUB_APP_PEM}"
10+
GITHUB_ORG="{GITHUB_ORG}"
11+
GITHUB_GROUP="{GITHUB_GROUP}"
12+
GITHUB_LABELS="{GITHUB_LABELS}"
13+
GITHUB_RUNNERS_COUNT="{GITHUB_RUNNERS_COUNT}"
14+
15+
get_github_runners_token() {
16+
NOW=$( date +%s )
17+
IAT=$(($NOW - 60))
18+
EXP=$(($NOW + 540))
19+
HEADER_RAW='{"alg":"RS256"}'
20+
HEADER=$( echo -n "$HEADER_RAW" | openssl base64 | tr -d '=' | tr '/+' '_-' | tr -d '\n' )
21+
PAYLOAD_RAW='{"iat":'"$IAT"',"exp":'"$EXP"',"iss":'"$GITHUB_APP_ID"'}'
22+
PAYLOAD=$( echo -n "$PAYLOAD_RAW" | openssl base64 | tr -d '=' | tr '/+' '_-' | tr -d '\n' )
23+
HEADER_PAYLOAD="$HEADER"."$PAYLOAD"
24+
25+
# Making a tmp directory here because /bin/sh doesn't support process redirection <()
26+
tmp_dir=/tmp/github_app_tmp
27+
mkdir "$tmp_dir"
28+
echo -n "$GITHUB_APP_PEM" > "$tmp_dir/github.pem"
29+
echo -n "$HEADER_PAYLOAD" > "$tmp_dir/header"
30+
SIGNATURE=$( openssl dgst -sha256 -sign "$tmp_dir/github.pem" "$tmp_dir/header" | openssl base64 | tr -d '=' | tr '/+' '_-' | tr -d '\n' )
31+
rm -rf "$tmp_dir"
32+
33+
JWT="$HEADER_PAYLOAD"."$SIGNATURE"
34+
INSTALL_URL="https://$GITHUB_API_SERVER/app/installations/$GITHUB_APP_INSTALL_ID/access_tokens"
35+
INSTALL_TOKEN_PAYLOAD=$(curl -sSfLX POST -H "Authorization: Bearer $JWT" -H "Accept: application/vnd.github.v3+json" "$INSTALL_URL")
36+
INSTALL_TOKEN=$(echo $INSTALL_TOKEN_PAYLOAD | jq .token --raw-output)
37+
38+
token_url="https://$GITHUB_API_SERVER/orgs/$GITHUB_ORG/actions/runners/registration-token"
39+
payload=$(curl -sSfLX POST -H "Authorization: token $INSTALL_TOKEN" $token_url)
40+
41+
RUNNER_TOKEN=$(echo $payload | jq .token --raw-output)
42+
echo "$RUNNER_TOKEN"
43+
}
44+
45+
# set up dependencies
46+
sudo apt-get update || true
47+
sudo apt-get update
48+
sudo apt-get -y upgrade
49+
sudo apt-get install -y qemu-kvm jq docker.io docker-buildx make \
50+
clang cpp gcc zlib1g-dev libaio-dev libcap-dev libelf-dev \
51+
liburing-dev net-tools netcat-traditional socat \
52+
iptables software-properties-common netcat-openbsd iproute2
53+
54+
VERSION_GO_CONTAINERREGISTRY=v0.19.1
55+
URL="https://github.com/google/go-containerregistry/releases/download/$VERSION_GO_CONTAINERREGISTRY/go-containerregistry_Linux_arm64.tar.gz"
56+
curl -fSL $URL | sudo tar -xz -C /usr/local/bin crane
57+
crane version
58+
59+
# set up SSH for LVH
60+
sudo ufw allow OpenSSH
61+
sudo ufw --force enable
62+
sudo sed -i "s/.*PasswordAuthentication .*/PasswordAuthentication no/g" /etc/ssh/sshd_config
63+
sudo sed -i "s/.*PubkeyAuthentication .*/PubkeyAuthentication yes/g" /etc/ssh/sshd_config
64+
sudo systemctl restart ssh
65+
66+
# prepare multi-runners
67+
mkdir -p multi-runners && cd multi-runners
68+
VERSION_MULTI_RUNNERS=v1.2.0
69+
curl -L -o multi-runners.tar.gz https://github.com/vbem/multi-runners/archive/refs/tags/$VERSION_MULTI_RUNNERS.tar.gz
70+
tar xzf ./multi-runners.tar.gz --strip-components=1
71+
72+
# get GH token for runners provisioning
73+
GITHUB_RUNNERS_TOKEN=$(get_github_runners_token)
74+
75+
# create runners
76+
sudo tee .env <<EOF
77+
MR_RELEASE_URL='https://github.com/actions/runner/releases/download/v2.321.0/actions-runner-linux-arm64-2.321.0.tar.gz'
78+
EOF
79+
for (( i=1; i<=$GITHUB_RUNNERS_COUNT; i++ ))
80+
do
81+
./mr.bash add --org $GITHUB_ORG --group $GITHUB_GROUP --labels $GITHUB_LABELS --user runner-$i --token $GITHUB_RUNNERS_TOKEN
82+
done
83+
84+
# prepare directories for LVH
85+
mkdir -p /home/runners && chown :runners /home/runners && chmod 774 /home/runners
86+

0 commit comments

Comments
 (0)