Skip to content

Commit a4035c4

Browse files
johnstairsjohnstairs
authored
Speed up release pipeline (#319)
1 parent 0414a0b commit a4035c4

7 files changed

Lines changed: 145 additions & 68 deletions

File tree

‎.github/workflows/delete-pr-environments.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ jobs:
3232
- name: Log in to Azure
3333
uses: ./.github/actions/login-azure
3434

35-
- uses: actions/setup-go@v4
35+
- uses: actions/setup-go@v6
3636
with:
3737
go-version-file: cli/go.mod
3838
cache-dependency-path: cli/go.sum

‎.github/workflows/pr-ci.yml‎

Lines changed: 11 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@ name: Tyger
33
on:
44
push:
55
branches: [main]
6-
tags: ["v*.*.*"]
76
pull_request: # all branches
87
workflow_dispatch:
98

@@ -64,12 +63,12 @@ jobs:
6463
with:
6564
fetch-depth: 0
6665

67-
- uses: actions/setup-go@v4
66+
- uses: actions/setup-go@v6
6867
with:
6968
go-version-file: cli/go.mod
7069
cache-dependency-path: cli/go.sum
7170

72-
- uses: actions/setup-dotnet@v3
71+
- uses: actions/setup-dotnet@v5
7372
with:
7473
global-json-file: server/global.json
7574

@@ -351,7 +350,7 @@ jobs:
351350
run: |
352351
make login-acr
353352
354-
- uses: actions/setup-go@v4
353+
- uses: actions/setup-go@v6
355354
with:
356355
go-version-file: cli/go.mod
357356
cache-dependency-path: cli/go.sum
@@ -382,7 +381,7 @@ jobs:
382381
- name: Log in to Azure
383382
uses: ./.github/actions/login-azure
384383

385-
- uses: actions/setup-go@v4
384+
- uses: actions/setup-go@v6
386385
with:
387386
go-version-file: cli/go.mod
388387
cache-dependency-path: cli/go.sum
@@ -425,7 +424,7 @@ jobs:
425424
- name: Log in to Azure
426425
uses: ./.github/actions/login-azure
427426

428-
- uses: actions/setup-go@v4
427+
- uses: actions/setup-go@v6
429428
with:
430429
go-version-file: cli/go.mod
431430
cache-dependency-path: cli/go.sum
@@ -465,7 +464,7 @@ jobs:
465464
run: |
466465
make login-acr
467466
468-
- uses: actions/setup-go@v4
467+
- uses: actions/setup-go@v6
469468
with:
470469
go-version-file: cli/go.mod
471470
cache-dependency-path: cli/go.sum
@@ -509,7 +508,7 @@ jobs:
509508
with:
510509
fetch-depth: 0
511510

512-
- uses: actions/setup-go@v4
511+
- uses: actions/setup-go@v6
513512
with:
514513
go-version-file: cli/go.mod
515514
cache-dependency-path: cli/go.sum
@@ -660,12 +659,12 @@ jobs:
660659
with:
661660
fetch-depth: 0
662661

663-
- uses: actions/setup-go@v4
662+
- uses: actions/setup-go@v6
664663
with:
665664
go-version-file: cli/go.mod
666665
cache-dependency-path: cli/go.sum
667666

668-
- uses: actions/setup-dotnet@v3
667+
- uses: actions/setup-dotnet@v5
669668
with:
670669
global-json-file: server/global.json
671670

@@ -696,13 +695,13 @@ jobs:
696695
with:
697696
fetch-depth: 0
698697

699-
- uses: actions/setup-go@v4
698+
- uses: actions/setup-go@v6
700699
if: matrix.language == 'go'
701700
with:
702701
go-version-file: cli/go.mod
703702
cache-dependency-path: cli/go.sum
704703

705-
- uses: actions/setup-dotnet@v3
704+
- uses: actions/setup-dotnet@v5
706705
if: matrix.language == 'csharp'
707706
with:
708707
global-json-file: server/global.json
@@ -771,50 +770,3 @@ jobs:
771770
- name: Deploy to GitHub Pages
772771
id: deployment
773772
uses: actions/deploy-pages@v4
774-
775-
release:
776-
if: startsWith(github.ref, 'refs/tags/')
777-
needs:
778-
- unit-tests-and-format
779-
- integration-tests
780-
- get-config
781-
- verify-docker
782-
- windows-smoke-tests
783-
environment:
784-
name: publish-container-images
785-
env:
786-
DEVELOPER_CONFIG_BASE64: ${{ needs.get-config.outputs.DEVELOPER_CONFIG_BASE64 }}
787-
permissions:
788-
contents: write
789-
runs-on: ubuntu-latest
790-
steps:
791-
- uses: actions/checkout@v6
792-
with:
793-
fetch-depth: 0
794-
795-
- uses: actions/setup-go@v4
796-
with:
797-
go-version-file: cli/go.mod
798-
cache-dependency-path: cli/go.sum
799-
800-
- name: get container registry
801-
run: |
802-
set -euo pipefail
803-
official_pull_container_registry=$(echo "$DEVELOPER_CONFIG_BASE64" | base64 -d | jq -r '.officialPullContainerRegistry.fqdn')
804-
echo "GORELEASER_CONTAINER_REGISTRY=$(echo $official_pull_container_registry)" >> $GITHUB_ENV
805-
official_pull_container_registry_directory=$(echo "$DEVELOPER_CONFIG_BASE64" | base64 -d | jq -r '.officialPullContainerRegistry.directory // ""')
806-
echo "GORELEASER_CONTAINER_REGISTRY_DIRECTORY=$(echo $official_pull_container_registry_directory)" >> $GITHUB_ENV
807-
808-
- name: test image published
809-
run: |
810-
set -euo pipefail
811-
docker pull "${GORELEASER_CONTAINER_REGISTRY}${GORELEASER_CONTAINER_REGISTRY_DIRECTORY}/tyger-server:$(git describe --tags)"
812-
813-
- name: Run GoReleaser
814-
uses: goreleaser/goreleaser-action@v5
815-
env:
816-
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
817-
with:
818-
version: v1.21.2
819-
workdir: cli
820-
args: release --clean

‎.github/workflows/release.yml‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
name: Release
2+
3+
on:
4+
push:
5+
tags: ["v*.*.*"]
6+
workflow_dispatch:
7+
8+
permissions:
9+
contents: write
10+
11+
jobs:
12+
release:
13+
runs-on: ubuntu-latest
14+
defaults:
15+
run:
16+
shell: bash
17+
steps:
18+
- uses: actions/checkout@v6
19+
with:
20+
fetch-depth: 0
21+
22+
- uses: actions/setup-go@v6
23+
with:
24+
go-version-file: cli/go.mod
25+
cache-dependency-path: cli/go.sum
26+
27+
- name: Get container registry
28+
run: |
29+
set -euo pipefail
30+
developer_config_json=$(scripts/get-config.sh --dev -o json | jq -c)
31+
official_pull_container_registry=$(jq -r '.officialPullContainerRegistry.fqdn' <<<"$developer_config_json")
32+
echo "GORELEASER_CONTAINER_REGISTRY=$official_pull_container_registry" >> "$GITHUB_ENV"
33+
official_pull_container_registry_directory=$(jq -r '.officialPullContainerRegistry.directory // ""' <<<"$developer_config_json")
34+
echo "GORELEASER_CONTAINER_REGISTRY_DIRECTORY=$official_pull_container_registry_directory" >> "$GITHUB_ENV"
35+
36+
- name: Run GoReleaser
37+
uses: goreleaser/goreleaser-action@v5
38+
env:
39+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
40+
with:
41+
version: v1.21.2
42+
workdir: cli
43+
args: release --clean
44+
45+
- name: Verify official artifacts published
46+
timeout-minutes: 35
47+
run: |
48+
set -euo pipefail
49+
50+
mapfile -t artifact_refs < <(EXPLICIT_IMAGE_TAG="$GITHUB_REF_NAME" make -s list-official-artifacts)
51+
if (( ${#artifact_refs[@]} == 0 )); then
52+
echo "ERROR: no official artifact references were generated."
53+
exit 1
54+
fi
55+
56+
max_attempts=31
57+
retry_delay_seconds=60
58+
59+
for attempt in $(seq 1 "$max_attempts"); do
60+
missing_artifacts=()
61+
62+
echo "Checking official artifact manifests (attempt ${attempt}/${max_attempts})..."
63+
64+
for artifact_ref in "${artifact_refs[@]}"; do
65+
echo "Checking $artifact_ref"
66+
if docker manifest inspect "$artifact_ref" >/dev/null 2>&1; then
67+
echo "Found $artifact_ref"
68+
else
69+
echo "Missing $artifact_ref"
70+
missing_artifacts+=("$artifact_ref")
71+
fi
72+
done
73+
74+
if (( ${#missing_artifacts[@]} == 0 )); then
75+
echo "All official image and Helm chart manifests are available."
76+
exit 0
77+
fi
78+
79+
if (( attempt == max_attempts )); then
80+
echo "ERROR: official artifacts were not available after 30 minutes."
81+
exit 1
82+
fi
83+
84+
echo "Official artifacts not ready. Attempt ${attempt}/${max_attempts}. ${#missing_artifacts[@]} missing artifact manifest(s). Retrying in 60 seconds."
85+
sleep "$retry_delay_seconds"
86+
done

‎Makefile‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -79,8 +79,12 @@ _docker-build:
7979
container_registry_spec='${CONTAINER_REGISTRY_SPEC}'
8080
registry=$$(echo "$${container_registry_spec}" | jq -r '.fqdn')
8181
directory=$$(echo "$${container_registry_spec}" | jq -r '.directory // ""')
82+
only_list_artifacts_arg=""
83+
if [[ "${DOCKER_BUILD_ONLY_LIST_ARTIFACTS}" == "true" ]]; then
84+
only_list_artifacts_arg="--only-list-artifacts"
85+
fi
8286

83-
scripts/build-images.sh $$target_arg ${DOCKER_BUILD_ARCH_FLAGS} ${DOCKER_BUILD_PUSH_FLAGS} --tag "$$tag" --registry "$${registry}" --registry-directory "$${directory}"
87+
scripts/build-images.sh $$target_arg ${DOCKER_BUILD_ARCH_FLAGS} ${DOCKER_BUILD_PUSH_FLAGS} $${only_list_artifacts_arg} --tag "$$tag" --registry "$${registry}" --registry-directory "$${directory}"
8488

8589
docker-build-test: login-acr
8690
$(MAKE) _docker-build DOCKER_BUILD_TARGET=test
@@ -99,7 +103,12 @@ docker-build: docker-build-test docker-build-server docker-build-client
99103
publish-official-images:
100104
container_registry_spec=$$(echo '${DEVELOPER_CONFIG_JSON}' | jq -c '.officialPushContainerRegistry')
101105
export EXPLICIT_IMAGE_TAG="$$(git describe --tags)"
102-
$(MAKE) DOCKER_BUILD_ARCH_FLAGS="--arch amd64 --arch arm64" CONTAINER_REGISTRY_SPEC="$${container_registry_spec}" docker-build-server docker-build-client docker-build-helm
106+
$(MAKE) DOCKER_BUILD_ARCH_FLAGS="--arch amd64 --arch arm64" DOCKER_BUILD_PUSH_FLAGS="--push --push-force" CONTAINER_REGISTRY_SPEC="$${container_registry_spec}" docker-build-server docker-build-client docker-build-helm
107+
108+
list-official-artifacts:
109+
container_registry_spec=$$(echo '${DEVELOPER_CONFIG_JSON}' | jq -c '.officialPullContainerRegistry')
110+
export EXPLICIT_IMAGE_TAG="$${EXPLICIT_IMAGE_TAG:-$$(git describe --tags)}"
111+
$(MAKE) DOCKER_BUILD_ONLY_LIST_ARTIFACTS="true" DOCKER_BUILD_ARCH_FLAGS="--arch amd64 --arch arm64" DOCKER_BUILD_PUSH_FLAGS="" CONTAINER_REGISTRY_SPEC="$${container_registry_spec}" docker-build-server docker-build-client docker-build-helm
103112

104113
publish-ghcr:
105114
container_registry_spec="{\"fqdn\": \"ghcr.io\", \"directory\": \"/$${GITHUB_REPOSITORY}\"}"

‎Makefile.cloud‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,10 @@ set-context:
6464
done
6565

6666
login-acr:
67+
if [[ "${DOCKER_BUILD_ONLY_LIST_ARTIFACTS}" == "true" ]]; then
68+
exit 0
69+
fi
70+
6771
registry=$$(echo '${CONTAINER_REGISTRY_SPEC}' | jq -r '.fqdn')
6872
if [[ "$${registry}" =~ \.azurecr\.io$$ ]]; then
6973
if ! ./scripts/check-docker-login.sh "$${registry}"; then

‎cli/.goreleaser.yaml‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -48,11 +48,7 @@ archives:
4848
format: zip
4949

5050
changelog:
51-
sort: asc
52-
filters:
53-
exclude:
54-
- "^docs:"
55-
- "^test:"
51+
use: github-native
5652

5753
release:
5854
draft: true

‎scripts/build-images.sh‎

Lines changed: 31 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ Options:
2222
--arch amd64|arm64 The architecture to build for. Can be specified multiple times.
2323
--push Push runtime images (requires --tag or --use-git-hash-as-tag)
2424
--push-force Force runtime images, will overwrite images with same tag (requires --tag or --use-git-hash-as-tag)
25+
--only-list-artifacts Print artifact references that would be published and exit without building or pushing
2526
--tag <tag> Tag for runtime images
2627
--use-git-hash-as-tag Use the current git hash as tag
2728
-h, --help Brings up this menu
@@ -78,6 +79,10 @@ while [[ $# -gt 0 ]]; do
7879
force=1
7980
shift
8081
;;
82+
--only-list-artifacts)
83+
only_list_artifacts=1
84+
shift
85+
;;
8186
--tag)
8287
image_tag="$2"
8388
shift 2
@@ -98,6 +103,11 @@ while [[ $# -gt 0 ]]; do
98103
esac
99104
done
100105

106+
if [[ -n "${only_list_artifacts:-}" && -n "${push:-}" ]]; then
107+
echo "ERROR: --only-list-artifacts cannot be used with --push or --push-force"
108+
exit 1
109+
fi
110+
101111
# Ensure registry_dir starts with a /
102112
if [[ ! "$registry_dir" =~ ^/ ]]; then
103113
registry_dir="/$registry_dir"
@@ -126,6 +136,12 @@ function build_and_push_platform() {
126136
local image_tag_with_platform="${image_tag}-${platform}"
127137

128138
full_image="${container_registry_fqdn}${registry_dir}${repo}:${image_tag_with_platform}"
139+
140+
if [[ -n "${only_list_artifacts:-}" ]]; then
141+
echo "$full_image"
142+
return 0
143+
fi
144+
129145
echo "Building image ${full_image}..."
130146

131147
set +e
@@ -169,12 +185,20 @@ function build_and_push() {
169185
build_and_push_platform "${build_context}" "${dockerfile_path}" "${repo}" "arm64"
170186
fi
171187

188+
manifest_name="${container_registry_fqdn}${registry_dir}${repo}:${image_tag}"
189+
190+
if [[ -n "${only_list_artifacts:-}" ]]; then
191+
if [[ -n "${amd64:-}" && -n "${arm64:-}" ]]; then
192+
echo "$manifest_name"
193+
fi
194+
return 0
195+
fi
196+
172197
# if not pushing or not building for both platforms, skip creating a manifest
173198
if [[ -z "${push:-}" || -z "${amd64:-}" || -z "${arm64:-}" ]]; then
174199
return 0
175200
fi
176201

177-
manifest_name="${container_registry_fqdn}${registry_dir}${repo}:${image_tag}"
178202
docker manifest create --amend "${manifest_name}" "${container_registry_fqdn}${registry_dir}${repo}:${image_tag}-amd64" "${container_registry_fqdn}${registry_dir}${repo}:${image_tag}-arm64" >/dev/null
179203

180204
# Push manigest
@@ -218,6 +242,12 @@ fi
218242

219243
if [[ -n "${helm:-}" ]]; then
220244

245+
helm_chart_ref="${container_registry_fqdn}${registry_dir}helm/tyger:${image_tag}"
246+
if [[ -n "${only_list_artifacts:-}" ]]; then
247+
echo "$helm_chart_ref"
248+
exit 0
249+
fi
250+
221251
# If we are pushing to an Azure Container Registry, we need to do a special login
222252
if [[ "${container_registry_fqdn}" =~ \.azurecr\.io$ ]]; then
223253
echo "logging in to ACR to publish helm chart..."

0 commit comments

Comments
 (0)