Skip to content

Commit b3cc425

Browse files
jmakar-s1jmakar-s1
authored
Misc GHA workflow build updates (#1333)
1 parent f5b1ac5 commit b3cc425

File tree

50 files changed

+501
-317
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

50 files changed

+501
-317
lines changed

‎.github/actions/setup-minikube-cluster/action.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ runs:
3030
- name: Create minikube Kubernetes ${{ inputs.k8s_version }} Cluster
3131
id: create-minikube-cluster
3232
if: ${{ inputs.k8s_version != '' && inputs.k8s_version != 'v1.24.0' }}
33-
uses: manusa/actions-setup-minikube@3856c6fa039819f1c8e7e248b1fc5a8564e354c9 # v2.9.0
33+
uses: manusa/actions-setup-minikube@v2.14.0
3434
with:
3535
minikube version: '${{ inputs.minikube_version }}'
3636
kubernetes version: '${{ inputs.k8s_version }}'

‎.github/workflows/retry-workflow-build-container-images.yaml‎

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -22,14 +22,15 @@ jobs:
2222
- name: Checkout repository
2323
uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4
2424

25-
- name: Retry workflow
26-
env:
27-
GH_TOKEN: ${{ github.token }}
28-
run: |
29-
echo "Giving the workflow a 5 minute head start"
30-
sleep 300
31-
echo "Retrying workflow"
32-
scripts/cicd/gha/retry_workflow.sh "Build Container Images"
25+
# TODO Why is this needed?
26+
#- name: Retry workflow
27+
# env:
28+
# GH_TOKEN: ${{ github.token }}
29+
# run: |
30+
# echo "Giving the workflow a 5 minute head start"
31+
# sleep 300
32+
# echo "Retrying workflow"
33+
# scripts/cicd/gha/retry_workflow.sh "Build Container Images"
3334

3435
- name: Notify Slack on Failure
3536
if: ${{ failure() && github.ref_name == 'master' }}

‎.github/workflows/retry-workflow-build-linux-packages.yaml‎

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -22,14 +22,15 @@ jobs:
2222
- name: Checkout repository
2323
uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4
2424

25-
- name: Retry workflow
26-
env:
27-
GH_TOKEN: ${{ github.token }}
28-
run: |
29-
echo "Giving the workflow a 5 minute head start"
30-
sleep 300
31-
echo "Retrying workflow"
32-
scripts/cicd/gha/retry_workflow.sh "Build Linux Packages"
25+
# TODO Why is this needed?
26+
#- name: Retry workflow
27+
# env:
28+
# GH_TOKEN: ${{ github.token }}
29+
# run: |
30+
# echo "Giving the workflow a 5 minute head start"
31+
# sleep 300
32+
# echo "Retrying workflow"
33+
# scripts/cicd/gha/retry_workflow.sh "Build Linux Packages"
3334

3435
- name: Notify Slack on Failure
3536
if: ${{ failure() && github.ref_name == 'master' }}

‎.github/workflows/reusable-agent-build-container-images.yml‎

Lines changed: 37 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ on:
1818
cicd_workflow:
1919
description: "Additional string that has to differentiate AWS object from another workflows."
2020
type: string
21-
21+
2222
builder_name:
2323
description: "Builder name used to build docker images. Accepted values: ubuntu, alpine, ubuntu-fips."
2424
type: string
@@ -187,7 +187,7 @@ jobs:
187187
with:
188188
username: ${{ secrets.PULL_REGISTRY_USERNAME }}
189189
password: ${{ secrets.PULL_REGISTRY_PASSWORD }}
190-
190+
191191
- name: Login to an internal build registry
192192
if: vars.INTERNAL_BUILD_REGISTRY
193193
uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3
@@ -204,7 +204,7 @@ jobs:
204204
with:
205205
image: tonistiigi/binfmt:qemu-v6.2.0
206206
platforms: all
207-
207+
208208
# Context need to be created for a self-hosted runner
209209
- name: Set up Docker Context for Buildx
210210
id: buildx-context
@@ -238,7 +238,7 @@ jobs:
238238
build-tarball \
239239
--image-type ${{ matrix.variant.image_type }} \
240240
--output-dir ./build
241-
241+
242242
- name: Save image OCI tarball as artifact
243243
uses: actions/upload-artifact@v4
244244
with:
@@ -249,7 +249,7 @@ jobs:
249249
# Jobs which performs basic sanity checks for the Kubernetes Monitor and Kubernetes Events Monitor
250250
k8s_kubernetes_monitor_tests:
251251
name: Kubernetes Monitors - k8s ${{ inputs.builder_name }} ${{ matrix.k8s_version.version }}-${{ matrix.k8s_version.runtime}}
252-
runs-on: ubuntu-22.04
252+
runs-on: ubuntu-24.04
253253
timeout-minutes: 15
254254

255255
if: ${{ ! vars.K8S_TESTS_DISABLED }}
@@ -321,7 +321,7 @@ jobs:
321321

322322
- name: Preload minikube images
323323
run: |
324-
# Since the minikube runtimes we use cannot be reasonably configured to use custom credentials for docker hub,
324+
# Since the minikube runtimes we use cannot be reasonably configured to use custom credentials for docker hub,
325325
# we need to pull the images and load them into minikube.
326326
docker pull busybox:1.28
327327
minikube image load busybox:1.28
@@ -385,7 +385,7 @@ jobs:
385385
minikube image ls
386386
kubectl apply -f tests/e2e/k8s_k8s_monitor/std_printer_deployment.yaml
387387
kubectl apply -f tests/e2e/k8s_k8s_monitor/long_message_printer_deployment.yaml
388-
388+
389389
kubectl wait --for=condition=ready pod -l app=std-printer
390390
kubectl wait --for=condition=ready pod -l app=long-message-printer
391391
kubectl get pods -A
@@ -396,7 +396,7 @@ jobs:
396396
397397
echo ""
398398
kubectl logs "${APP_POD_NAME}"
399-
echo ""
399+
echo ""
400400
401401
- name: Create mock pods for multi account printer
402402
env:
@@ -507,7 +507,7 @@ jobs:
507507
echo "Kubernetes event monitor log has to be handled by the agent 1 time, but it got ${k8s_event_log_files_number}"
508508
exit 1
509509
fi
510-
510+
511511
echo "Wait for BEGIN DELAYED MESSAGE-END DELAYED MESSAGE lines. They should not be splitted."
512512
MINIMUM_RESULTS=10 ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" app="std-printer" parser="test-parser-1" stream="stdout" "BEGIN DELAYED MESSAGE stdout"'
513513
MINIMUM_RESULTS=10 ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" app="std-printer" parser="test-parser-1" stream="stderr" "BEGIN DELAYED MESSAGE stderr"'
@@ -517,7 +517,7 @@ jobs:
517517
echo 'Looking for mixed streams'
518518
RETRY_ATTEMPTS=1 ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" app="std-printer" parser="test-parser-1" stream="stdout" "stderr"' && echo -e $MIXED_STREAMS_ERR_MSG && exit 1 || echo -e $MIXED_STREAMS_SUCCESS_MSG
519519
RETRY_ATTEMPTS=1 ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" app="std-printer" parser="test-parser-1" stream="stderr" "stdout"' && echo -e $MIXED_STREAMS_ERR_MSG && exit 1 || echo -e $MIXED_STREAMS_SUCCESS_MSG
520-
520+
521521
echo "Wait for BEGIN_LONG_MESSAGE .* END_LONG_MESSAGE lines. They should not be splitted."
522522
MINIMUM_RESULTS=10 ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" app="long-message-printer" parser="test-parser-2" stream="stdout" "stdout_BEGIN_LONG_MESSAGE"'
523523
MINIMUM_RESULTS=10 ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" app="long-message-printer" parser="test-parser-2" stream="stderr" "stderr_BEGIN_LONG_MESSAGE"'
@@ -528,7 +528,7 @@ jobs:
528528
RETRY_ATTEMPTS=1 ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" app="long-message-printer" parser="test-parser-2" stream="stdout" "stderr"' && echo -e $MIXED_STREAMS_ERR_MSG && exit 1 || echo -e $MIXED_STREAMS_SUCCESS_MSG
529529
RETRY_ATTEMPTS=1 ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" app="long-message-printer" parser="test-parser-2" stream="stderr" "stdout"' && echo -e $MIXED_STREAMS_ERR_MSG && exit 1 || echo -e $MIXED_STREAMS_SUCCESS_MSG
530530
531-
531+
532532
echo ""
533533
534534
# We only assert this under newer Kubernetes versions since in the old ones CronJobs were not
@@ -544,9 +544,9 @@ jobs:
544544
run: |
545545
export RETRY_ATTEMPTS="14"
546546
export SLEEP_DELAY="10"
547-
548-
# Wait for at least one hello-printer job to be scheduled and completed.
549-
RETRIES=1;
547+
548+
# Wait for at least one hello-printer job to be scheduled and completed.
549+
RETRIES=1;
550550
until echo -n "Waiting for first job to be completed - retry $RETRIES: " && kubectl wait --for=condition=complete job -l app=hello-printer || [ $RETRIES -gt 60 ]; do sleep 1; RETRIES=$(($RETRIES+1)); done
551551
[ $RETRIES -gt 60 ] && echo "Job didn't start in time" && exit 1
552552
@@ -556,7 +556,7 @@ jobs:
556556
557557
echo "CronJob events checks"
558558
./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile="/var/log/scalyr-agent-2/kubernetes_events.log" $parser="k8sEvents" k8s-kind="CronJob" involvedObjectKind="CronJob" involvedObjectName="hello" watchEventType="ADDED" reason="SawCompletedJob"'
559-
559+
560560
./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" "Hello from the Kubernetes cluster"'
561561
562562
- name: Check the short lived pods logs are ingested - 404 response handling
@@ -569,16 +569,16 @@ jobs:
569569
./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" message matches "Pod default/short-lived-job-1.* not found in K8s API. Including pod based on SCALYR_K8S_INCLUDE_ALL_CONTAINERS=true."'
570570
./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" message matches "Pod default/short-lived-job-2.* not found in K8s API. Including pod based on SCALYR_K8S_INCLUDE_ALL_CONTAINERS=true."'
571571
./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" message matches "Pod default/short-lived-job-3.* not found in K8s API. Including pod based on SCALYR_K8S_INCLUDE_ALL_CONTAINERS=true."'
572-
572+
573573
- name: Check the short lived pods logs are ingested - log message
574574
env:
575575
# Needed for scalyr-tool
576576
scalyr_readlog_token: "${{ secrets.CT_SCALYR_TOKEN_PROD_US_CLOUDTECH_TESTING_READ }}"
577577
SCALYR_AGENT_POD_NAME: "${{ env.SCALYR_AGENT_POD_NAME }}"
578578
run: |
579-
./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" "Message from short-lived-container-1"'
580-
./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" "Message from short-lived-container-2"'
581-
./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" "Message from short-lived-container-3"'
579+
RETRY_ATTEMPTS=20 ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" "Message from short-lived-container-1"'
580+
RETRY_ATTEMPTS=20 ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" "Message from short-lived-container-2"'
581+
RETRY_ATTEMPTS=20 ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" "Message from short-lived-container-3"'
582582
583583
- name: Install Testing Python Packages
584584
run: |
@@ -604,13 +604,13 @@ jobs:
604604
ACCOUNT_NAME_7: "[email protected]"
605605
run: |
606606
echo 'Looking for multiple account logs'
607-
607+
608608
./scripts/cicd/verify-multiaccount-records-have-been-ingested.py
609609
610610
611611
k8s_open_metrics_monitor_tests:
612612
name: OpenMetrics Monitor - k8s ${{ inputs.builder_name }} ${{ matrix.k8s_version.version }}-${{ matrix.k8s_version.runtime}}
613-
runs-on: ubuntu-22.04
613+
runs-on: ubuntu-24.04
614614
if: vars.K8S_TESTS_DISABLED != 'true' && github.ref_name != '_release_build' && inputs.builder_name != 'alpine'
615615
timeout-minutes: 15
616616

@@ -693,7 +693,7 @@ jobs:
693693
--image-type k8s
694694
695695
docker image ls
696-
696+
697697
# Needed for containerd or cri-o runtime
698698
if [ "${{ matrix.k8s_version.runtime }}" = "containerd" ] || [ "${{ matrix.k8s_version.runtime }}" = "cri-o" ]; then
699699
docker tag k8s-image:test docker.io/library/k8s-image:test
@@ -732,10 +732,10 @@ jobs:
732732
sleep 2
733733
kubectl wait --for=condition=ready pod -l app=node-exporter --namespace monitoring ||
734734
(echo "Failed to wait for node-exporter pod to become ready"; kubectl describe pods -l app=node-exporter --namespace monitoring; echo; echo GET PODS:; kubectl get pods -A; echo LOGS:; kubectl logs -l app=node-exporter --namespace monitoring; exit 1)
735-
735+
736736
kubectl wait --for=condition=ready pod -l app=java-hello-world ||
737737
(echo "Failed to wait for java-hello-world pod to become ready"; kubectl describe pods -l app=java-hello-world; echo; echo GET PODS:; kubectl get pods -A; echo LOGS:; kubectl logs -l app=java-hello-world; exit 1)
738-
738+
739739
kubectl get pods -A
740740
741741
- name: Create scalyr-agent-2 daemonset
@@ -760,7 +760,7 @@ jobs:
760760
run: |
761761
export RETRY_ATTEMPTS="30"
762762
export SLEEP_DELAY="10"
763-
763+
764764
# Verify agent is running
765765
echo "Agent running checks"
766766
./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile="/var/log/scalyr-agent-2/agent.log" "Starting scalyr agent..."'
@@ -833,7 +833,7 @@ jobs:
833833
- build_image_tarball
834834
name: K8s Smoketest - ${{ inputs.builder_name }}
835835
if: vars.K8S_TESTS_DISABLED != 'true' && github.ref_name != '_release_build' && (github.event_name != 'pull_request' || ! github.event.pull_request.draft)
836-
steps:
836+
steps:
837837
- name: Checkout Repository
838838
uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4
839839

@@ -858,8 +858,8 @@ jobs:
858858
- name: Set up Docker Context for Buildx
859859
id: buildx-context
860860
run: |
861-
docker context create builders || echo Already exists
862-
861+
docker context create builders || echo Already exists
862+
863863
- name: Set up Docker Buildx
864864
uses: docker/setup-buildx-action@v3
865865
with:
@@ -904,7 +904,7 @@ jobs:
904904
SCALYR_API_KEY: ${{ secrets.CT_SCALYR_TOKEN_PROD_US_CLOUDTECH_TESTING_WRITE }}
905905
READ_API_KEY: ${{ secrets.CT_SCALYR_TOKEN_PROD_US_CLOUDTECH_TESTING_READ }}
906906
run: |
907-
source ./.circleci/smoketest_k8s.sh scalyr/scalyr-agent-ci-unittest:4 150 no_delete_existing_k8s_objs k8s-image:test
907+
source ./.circleci/smoketest_k8s.sh scalyr/scalyr-agent-ci-unittest:4 300 no_delete_existing_k8s_objs k8s-image:test
908908
909909
- name: Upload test results
910910
uses: actions/upload-artifact@v4
@@ -925,11 +925,11 @@ jobs:
925925
matrix:
926926
variant:
927927
- image_type: docker-json
928-
timeout: 150
928+
timeout: 300
929929
- image_type: docker-syslog
930-
timeout: 150
930+
timeout: 300
931931
- image_type: docker-api
932-
timeout: 200
932+
timeout: 300
933933
steps:
934934
- name: Checkout Repository
935935
uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4
@@ -951,7 +951,7 @@ jobs:
951951
with:
952952
username: ${{ secrets.PULL_REGISTRY_USERNAME }}
953953
password: ${{ secrets.PULL_REGISTRY_PASSWORD }}
954-
954+
955955
- name: Login to an internal build registry
956956
if: vars.INTERNAL_BUILD_REGISTRY
957957
uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3
@@ -1025,7 +1025,7 @@ jobs:
10251025

10261026
publish_images:
10271027
name: Publish image ${{ matrix.variant.image_type }}-${{ inputs.builder_name }}
1028-
runs-on: ubuntu-22.04
1028+
runs-on: ubuntu-24.04
10291029
if: github.ref_type == 'tag' || github.ref_name == 'master' || github.head_ref == 'release' || github.event_name == 'pull_request'
10301030
needs:
10311031
# k8s_kubernetes_monitor_tests, k8s_open_metrics_monitor_tests, k8s-smoketest checks are not needed before pushing a test version to dockerhub.
@@ -1066,7 +1066,7 @@ jobs:
10661066
echo "tags=$AGENT_VERSION,latest" >> "${GITHUB_OUTPUT}"
10671067
else
10681068
echo "tags=empty" >> "${GITHUB_OUTPUT}"
1069-
fi
1069+
fi
10701070
elif [ "${{ github.ref_name }}" = "master" ]; then
10711071
echo "tags=${{ github.sha }}" >> "${GITHUB_OUTPUT}"
10721072
elif [ "${{ github.head_ref }}" = "release" ]; then
@@ -1079,7 +1079,7 @@ jobs:
10791079
fi
10801080
10811081
- name: Set up skopeo
1082-
uses: warjiang/setup-skopeo@latest
1082+
uses: warjiang/setup-skopeo@main
10831083
with:
10841084
version: latest
10851085

@@ -1092,6 +1092,7 @@ jobs:
10921092
--base-image ${{ inputs.base_image }} \
10931093
publish \
10941094
--registry docker.io \
1095+
--no-verify-tls \
10951096
--name-prefix "${{ secrets.PUBLISH_REGISTRY_USERNAME }}" \
10961097
--tags "${{ steps.verify.outputs.tags }}" \
10971098
--from-oci-layout /tmp/image_tarball/${{ matrix.variant.image_type }}-${{ inputs.builder_name }}.tar \

‎.github/workflows/reusable-agent-build-linux-packages-new.yml‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ jobs:
5555
matrix:
5656
builder:
5757
- { "name": "aio-aarch64", runner: "aws-aarch64" }
58-
- { "name": "aio-x86_64", runner: "ubuntu-22.04" }
58+
- { "name": "aio-x86_64", runner: "ubuntu-24.04" }
5959

6060
steps:
6161
- name: Checkout repository
@@ -113,12 +113,12 @@ jobs:
113113
strategy:
114114
matrix:
115115
variant:
116-
- { "builder_name": "aio-x86_64", "package_type": "deb", runner: "ubuntu-22.04" }
117-
- { "builder_name": "aio-x86_64", "package_type": "rpm", runner: "ubuntu-22.04" }
116+
- { "builder_name": "aio-x86_64", "package_type": "deb", runner: "ubuntu-24.04" }
117+
- { "builder_name": "aio-x86_64", "package_type": "rpm", runner: "ubuntu-24.04" }
118118
- { "builder_name": "aio-aarch64", "package_type": "deb", runner: "aws-aarch64" }
119119
- { "builder_name": "aio-aarch64", "package_type": "rpm", runner: "aws-aarch64" }
120-
- { "builder_name": "non-aio", "package_type": "deb", runner: "ubuntu-22.04" }
121-
- { "builder_name": "non-aio", "package_type": "rpm", runner: "ubuntu-22.04" }
120+
- { "builder_name": "non-aio", "package_type": "deb", runner: "ubuntu-24.04" }
121+
- { "builder_name": "non-aio", "package_type": "rpm", runner: "ubuntu-24.04" }
122122
steps:
123123
- name: Checkout repository
124124
uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4

‎.github/workflows/unittests.yml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ jobs:
5757
- os: windows-2022
5858
platform: windows
5959
python-version: "3.10"
60-
- os: macos-12
60+
- os: macos-13
6161
platform: macos
6262
python-version: "3.10"
6363
steps:
@@ -193,7 +193,7 @@ jobs:
193193
platform:
194194
- ubuntu
195195
include:
196-
- os: macos-12
196+
- os: macos-13
197197
platform: macos
198198
python-version: "3.10"
199199

@@ -230,7 +230,7 @@ jobs:
230230
- name: Upload pytest test results
231231
uses: actions/upload-artifact@v4
232232
with:
233-
name: pytest-results-${{ matrix.python-version }}-${{ matrix.variant }}
233+
name: pytest-results-${{ matrix.python-version }}-${{ matrix.platform }}-agent-source
234234
path: |
235235
test-results/junit*.xml
236236
if: ${{ success() || failure() }}

0 commit comments

Comments
 (0)