Skip to content
.github/workflows/reusable-agent-build-container-images.yml

Alternate ncurses download url #5

Sign in to view logs

Alternate ncurses download url

Alternate ncurses download url #5

Workflow file for this run

.github/workflows/reusable-agent-build-container-images.yml at 89d1d67
name: "Kubernetes End to End Tests"

Check failure on line 1 in .github/workflows/reusable-agent-build-container-images.yml

View workflow run for this annotation

GitHub Actions / .github/workflows/reusable-agent-build-container-images.yml

Invalid workflow file 

(Line: 1036, Col: 9): Job 'publish_images' depends on unknown job 'docker-smoketest'.
on:
workflow_call:
inputs:
python_version:
description: "Version of Python to use."
type: string
cache_version:
description: "Version suffix for the cache key. May be useful when it is needed to invalidate the cache."
type: string
aws_region:
description: "AWS region for remote builds."
type: string
cicd_workflow:
description: "Additional string that has to differentiate AWS object from another workflows."
type: string
builder_name:
description: "Builder name used to build docker images. Accepted values: ubuntu, alpine, ubuntu-fips."
type: string
base_image:
description: "Base image to be used when bulding the docker image, i.e. ubuntu:22.04, alpine:latest."
type: string
architectures:
description: "JSON representation of a list of architectures used in a stragety matrix in some jobs. Accepted values: x86_64, aarch64, armv7."
type: string
secrets:
CT_AWS_DEV_EC2_PRIVATE_KEY:
required: true
CT_AWS_DEV_EC2_PRIVATE_KEY_NAME:
required: true
CT_AWS_DEV_EC2_ACCESS_KEY:
required: true
CT_AWS_DEV_EC2_SECRET_KEY:
required: true
CT_SCALYR_TOKEN_PROD_US_CLOUDTECH_TESTING_WRITE:
required: true
CT_SCALYR_TOKEN_PROD_US_CLOUDTECH_TESTING_READ:
required: true
SCALYR_API_KEY_READ_2:
required: true
SCALYR_API_KEY_READ_3:
required: true
SCALYR_API_KEY_READ_4:
required: true
SCALYR_API_KEY_READ_5:
required: true
SCALYR_API_KEY_READ_6:
required: true
SCALYR_API_KEY_READ_7:
required: true
SCALYR_API_KEY_WRITE_2:
required: true
SCALYR_API_KEY_WRITE_3:
required: true
SCALYR_API_KEY_WRITE_4:
required: true
SCALYR_API_KEY_WRITE_5:
required: true
SCALYR_API_KEY_WRITE_6:
required: true
SCALYR_API_KEY_WRITE_7:
required: true
PULL_REGISTRY_USERNAME:
required: true
PULL_REGISTRY_PASSWORD:
required: true
PUBLISH_REGISTRY_USERNAME:
required: true
PUBLISH_REGISTRY_PASSWORD:
required: true
INTERNAL_BUILD_REGISTRY_USERNAME:
required: true
INTERNAL_BUILD_REGISTRY_PASSWORD:
required: true
jobs:
# Build cacheable requirements of the images.
build_image_requirements:
name: requirements-${{ inputs.builder_name }}-${{ matrix.architecture }}
runs-on: ${{ vars.RUNNER }}
strategy:
matrix:
architecture: ${{ fromJson(inputs.architectures) }}
steps:
- name: Checkout repository
uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4
- name: install python and requirements
uses: ./.github/actions/install_python_and_requirements
with:
python_version: ${{ inputs.python_version }}
- name: Login to DockerHub
uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3
with:
username: ${{ secrets.PULL_REGISTRY_USERNAME }}
password: ${{ secrets.PULL_REGISTRY_PASSWORD }}
- name: Login to an internal build registry
if: vars.INTERNAL_BUILD_REGISTRY
uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3
continue-on-error: true
with:
registry: ${{ vars.INTERNAL_BUILD_REGISTRY }}
username: ${{ secrets.INTERNAL_BUILD_REGISTRY_USERNAME }}
password: ${{ secrets.INTERNAL_BUILD_REGISTRY_PASSWORD }}
- name: Set up QEMU
id: qemu
uses: docker/setup-qemu-action@v3
with:
image: tonistiigi/binfmt:qemu-v6.2.0
platforms: all
# Context need to be created for a self-hosted runner
- name: Set up Docker Context for Buildx
id: buildx-context
run: |
docker context create builders || echo Already exists
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
with:
driver-opts: network=host
endpoint: builders
# TODO Investigate, no longer working
#- name: Expose GitHub Runtime To Be Able to Use GHA Cache By Docker.
# uses: crazy-max/ghaction-github-runtime@715c25b40ccc0df9b62bfa8be3ccc57d09dbc4b1
- name: Build requirements
shell: bash
id: build
env:
AWS_ACCESS_KEY: ${{ secrets.CT_AWS_DEV_EC2_ACCESS_KEY }}
AWS_SECRET_KEY: ${{ secrets.CT_AWS_DEV_EC2_SECRET_KEY }}
AWS_PRIVATE_KEY: ${{ secrets.CT_AWS_DEV_EC2_PRIVATE_KEY }}
AWS_PRIVATE_KEY_NAME: ${{ secrets.CT_AWS_DEV_EC2_PRIVATE_KEY_NAME }}
AWS_REGION: ${{ inputs.aws_region }}
CICD_WORKFLOW: ${{ inputs.cicd_workflow }}
CICD_JOB: build_image_requirements-${{ inputs.builder_name }}-${{ matrix.architecture }}
USE_GHA_CACHE: "1"
ALLOW_FALLBACK_TO_REMOTE_BUILDER: "1"
CACHE_VERSION: ${{ inputs.cache_version }}
run: |
python3 build_package_new.py \
image \
${{ inputs.builder_name }} \
--base-image ${{ inputs.base_image }} \
cache-requirements \
--architecture ${{ matrix.architecture }}
build_image_tarball:
name: Build image tarball ${{ matrix.variant.image_type }}-${{ inputs.builder_name }}
runs-on: ${{ vars.RUNNER }}
needs:
- build_image_requirements
strategy:
matrix:
variant:
- { "image_type": "k8s" }
- { "image_type": "docker-json" }
- { "image_type": "docker-syslog" }
- { "image_type": "docker-api" }
steps:
- name: Checkout repository
uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4
- name: install python and requirements
uses: ./.github/actions/install_python_and_requirements
with:
python_version: ${{ inputs.python_version }}
- name: Login to DockerHub
uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3
with:
username: ${{ secrets.PULL_REGISTRY_USERNAME }}
password: ${{ secrets.PULL_REGISTRY_PASSWORD }}
- name: Login to an internal build registry
if: vars.INTERNAL_BUILD_REGISTRY
uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3
continue-on-error: true
with:
registry: ${{ vars.INTERNAL_BUILD_REGISTRY }}
username: ${{ secrets.INTERNAL_BUILD_REGISTRY_USERNAME }}
password: ${{ secrets.INTERNAL_BUILD_REGISTRY_PASSWORD }}
- name: Set up QEMU
id: qemu
uses: docker/setup-qemu-action@v3
with:
image: tonistiigi/binfmt:qemu-v6.2.0
platforms: all
# Context need to be created for a self-hosted runner
- name: Set up Docker Context for Buildx
id: buildx-context
run: |
docker context create builders || echo Already exists
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
with:
driver-opts: network=host
endpoint: builders
# TODO Investigate, no longer working
#- name: Expose GitHub Runtime To Be Able to Use GHA Cache By Docker.
# uses: crazy-max/ghaction-github-runtime@715c25b40ccc0df9b62bfa8be3ccc57d09dbc4b1
- name: Build image tarball
shell: bash
id: build
env:
CICD_WORKFLOW: ${{ inputs.cicd_workflow }}
CICD_JOB: ${{ matrix.variant.image_type }}-${{ inputs.builder_name }}
USE_GHA_CACHE: "1"
CACHE_VERSION: ${{ inputs.cache_version }}
run: |
python3 build_package_new.py \
image \
${{ inputs.builder_name }} \
--base-image ${{ inputs.base_image }} \
build-tarball \
--image-type ${{ matrix.variant.image_type }} \
--output-dir ./build
- name: Save image OCI tarball as artifact
uses: actions/upload-artifact@v4
with:
name: container-image-tarball-${{ matrix.variant.image_type }}-${{ inputs.builder_name }}
path: build
retention-days: 1
# Jobs which performs basic sanity checks for the Kubernetes Monitor and Kubernetes Events Monitor
k8s_kubernetes_monitor_tests:
name: Kubernetes Monitors - k8s ${{ inputs.builder_name }} ${{ matrix.k8s_version.version }}-${{ matrix.k8s_version.runtime}}
runs-on: ubuntu-24.04
timeout-minutes: 15
if: ${{ ! vars.K8S_TESTS_DISABLED }}
needs:
- build_image_tarball
strategy:
fail-fast: false
matrix:
k8s_version:
# NOTE: We don't test under very old versions where CronJob was part of v2alpha1 since in
# those versions CronJobs are not enabled by default and need to be installed extra
- { "version": "v1.20.15", "driver": "", "runtime": "docker", "run_cronjob_checks": false }
- { "version": "v1.21.10", "driver": "", "runtime": "docker", "run_cronjob_checks": true }
- { "version": "v1.22.7", "driver": "", "runtime": "docker", "run_cronjob_checks": true }
# NOTE: Using containerd runtime in minikube on GHA only works with docker driver
- { "version": "v1.23.17", "driver": "docker", "runtime": "containerd", "run_cronjob_checks": true }
- { "version": "v1.24.14", "driver": "docker", "runtime": "containerd", "run_cronjob_checks": true }
- { "version": "v1.25.10", "driver": "docker", "runtime": "containerd", "run_cronjob_checks": true }
- { "version": "v1.26.5", "driver": "docker", "runtime": "containerd", "run_cronjob_checks": true }
# CRI-O
- { "version": "v1.27.4", "driver": "docker", "runtime": "cri-o", "run_cronjob_checks": true }
- { "version": "v1.29.3", "driver": "docker", "runtime": "cri-o", "run_cronjob_checks": true }
- { "version": "v1.30.0", "driver": "docker", "runtime": "cri-o", "run_cronjob_checks": true }
steps:
- name: Checkout Repository
uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4
with:
fetch-depth: 0
- name: install python and requirements
uses: ./.github/actions/install_python_and_requirements
with:
python_version: ${{ inputs.python_version }}
- name: Set up QEMU
id: qemu
uses: docker/setup-qemu-action@v3
with:
image: tonistiigi/binfmt:qemu-v6.2.0
platforms: all
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
with:
driver-opts: network=host
- name: Login to DockerHub
uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3
with:
username: ${{ secrets.PULL_REGISTRY_USERNAME }}
password: ${{ secrets.PULL_REGISTRY_PASSWORD }}
- name: Install Scalyr tool
run: |
curl https://raw.githubusercontent.com/scalyr/scalyr-tool/master/scalyr > scalyr
chmod +x scalyr
sudo mv scalyr /usr/local/bin
- name: Setup minikube k8s cluster
uses: ./.github/actions/setup-minikube-cluster/
with:
k8s_version: "${{ matrix.k8s_version.version }}"
minikube_driver: "${{ matrix.k8s_version.driver }}"
container_runtime: "${{ matrix.k8s_version.runtime }}"
github_token: "${{ secrets.GITHUB_TOKEN }}"
- name: Preload minikube images
run: |
# Since the minikube runtimes we use cannot be reasonably configured to use custom credentials for docker hub,
# we need to pull the images and load them into minikube.
docker pull busybox:1.28
minikube image load busybox:1.28
- name: Download OCI tarball of the built image.
uses: actions/download-artifact@v4
with:
name: container-image-tarball-k8s-${{ inputs.builder_name }}
path: /tmp/image_tarball
# TODO Investigate, no longer working
#- name: Expose GitHub Runtime To Be Able to Use GHA Cache By Docker.
# uses: crazy-max/ghaction-github-runtime@715c25b40ccc0df9b62bfa8be3ccc57d09dbc4b1
- name: Build Agent k8s Test Docker Image
env:
USE_GHA_CACHE: "1"
CACHE_VERSION: ${{ inputs.cache_version }}
run: |
python3 tests/end_to_end_tests/container_images_test/scripts/build_test_image.py \
--image-builder-name ${{ inputs.builder_name }} \
--base-image ${{ inputs.base_image }} \
--architecture x86_64 \
--image-oci-tarball /tmp/image_tarball/k8s-${{ inputs.builder_name }}.tar \
--result-image-name k8s-image:test \
--image-type k8s
docker image ls
# Needed for containerd or cri-o runtime
if [ "${{ matrix.k8s_version.runtime }}" = "containerd" ] || [ "${{ matrix.k8s_version.runtime }}" = "cri-o" ]; then
docker tag k8s-image:test docker.io/library/k8s-image:test
minikube image load docker.io/library/k8s-image:test
fi
- name: Upload Agent k8s Test Docker Image as artifact for debugging purposses
uses: actions/upload-artifact@v4
with:
name: k8s-image-${{ inputs.builder_name }}
path: build
retention-days: 1
# Here we build the dummy container which continuously prints data to stdout and stderr
- name: Build Dummy App Docker Images
run: |
docker build -f docker/Dockerfile.docker_monitor_testing_config -t std-printer scripts/
docker build -f docker/Dockerfile.long_message_printer -t long-message-printer scripts/
docker image ls
# Needed for containerd or cri-o runtime
if [ "${{ matrix.k8s_version.runtime }}" = "containerd" ] || [ "${{ matrix.k8s_version.runtime }}" = "cri-o" ]; then
for IMAGE in std-printer long-message-printer; do
docker tag ${IMAGE}:latest docker.io/library/${IMAGE}:latest
minikube image load docker.io/library/${IMAGE}:latest
done
fi
# Create pod for our mock std printer container which logs will be ingested by the agent
- name: Create mock pods
run: |
minikube image ls
kubectl apply -f tests/e2e/k8s_k8s_monitor/std_printer_deployment.yaml
kubectl apply -f tests/e2e/k8s_k8s_monitor/long_message_printer_deployment.yaml
kubectl wait --for=condition=ready pod -l app=std-printer
kubectl wait --for=condition=ready pod -l app=long-message-printer
kubectl get pods -A
export APP_POD_NAME=$(kubectl get pod --namespace=default --selector=app=std-printer -o jsonpath="{.items[0].metadata.name}")
echo "APP_POD_NAME=${APP_POD_NAME}" >> ${GITHUB_ENV}
echo "APP_POD_NAME=${APP_POD_NAME}"
echo ""
kubectl logs "${APP_POD_NAME}"
echo ""
- name: Create mock pods for multi account printer
env:
scalyr_api_key_write_team_2: "${{ secrets.SCALYR_API_KEY_WRITE_2 }}"
scalyr_api_key_write_team_3: "${{ secrets.SCALYR_API_KEY_WRITE_3 }}"
scalyr_api_key_write_team_4: "${{ secrets.SCALYR_API_KEY_WRITE_4 }}"
scalyr_api_key_write_team_5: "${{ secrets.SCALYR_API_KEY_WRITE_5 }}"
scalyr_api_key_write_team_6: "${{ secrets.SCALYR_API_KEY_WRITE_6 }}"
scalyr_api_key_write_team_7: "${{ secrets.SCALYR_API_KEY_WRITE_7 }}"
run: |
tests/e2e/k8s_k8s_monitor/multiple_account_printers/deploy.sh
# We create mock CronJob so we can assert that Kubernetes Events monitor correctly handles
# CronJob and Job Events.
- name: Create mock CronJob (v1)
if: ${{ matrix.k8s_version.run_cronjob_checks }}
run: |
# We need to use a different file for different K8s versions due to promotion (removal of
# old v1beta alias) from v1beta to v1 in v1.25.0. v1 has been available since v1.21.0
kubectl apply -f tests/e2e/k8s_events_monitor/cronjob_v1.yaml
kubectl get cronjob -A
- name: Wait random time to make it easier for the cloud github runners
run: sleep $((RANDOM % 60))
- name: Create scalyr-agent-2 daemonset
uses: ./.github/actions/install-k8s-agent/
with:
scalyr_server: "agent.scalyr.com"
scalyr_api_key: "${{ secrets.CT_SCALYR_TOKEN_PROD_US_CLOUDTECH_TESTING_WRITE }}"
scalyr_api_key_team_2: "${{ secrets.SCALYR_API_KEY_WRITE_2 }}"
scalyr_api_key_team_3: "${{ secrets.SCALYR_API_KEY_WRITE_3 }}"
scalyr_api_key_team_4: "${{ secrets.SCALYR_API_KEY_WRITE_4 }}"
scalyr_cluster_name: "${K8S_CLUSTER_NAME}"
scalyr_k8s_events_disable: "false"
main_yaml_path: "tests/e2e/scalyr-agent-2-daemonset.yaml"
- name: Create mock Jobs
run: |
kubectl apply -f tests/e2e/k8s_k8s_monitor/short-lived-jobs.yaml
kubectl get job -A
- name: Verify data has been ingested
timeout-minutes: 14
env:
# Needed for scalyr-tool
scalyr_readlog_token: "${{ secrets.CT_SCALYR_TOKEN_PROD_US_CLOUDTECH_TESTING_READ }}"
SCALYR_AGENT_POD_NAME: "${{ env.SCALYR_AGENT_POD_NAME }}"
K8S_NODE_NAME: "${{ env.K8S_NODE_NAME }}"
run: |
export RETRY_ATTEMPTS="14"
export SLEEP_DELAY="10"
# Verify agent is running
echo "Agent running checks"
./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile="/var/log/scalyr-agent-2/agent.log" "Starting scalyr agent..."'
# Verify Kubernetes monitor is running
echo "Kubernetes Monitor running checks"
./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile="/var/log/scalyr-agent-2/agent.log" "kubernetes_monitor parameters: ignoring namespaces: "'
echo ""
# Verify Kubernetes events monitor is running
echo "Kubernetes events monitor running checks"
./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile="/var/log/scalyr-agent-2/agent.log" "Starting monitor kubernetes_events_monitor"'
./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile="/var/log/scalyr-agent-2/agent.log" "Acting as Kubernetes event leader"'
echo ""
# Verify initial std-printer pod data has been ingested
# 1. First we want for some data to be ingested using "log.config.scalyr.com/attributes.parser"
# annotation set as part of the deployment YAML.
# After a while, we change that dynamically using kubectl and verify that this change has
# been correctly picked up by the agent.
echo Waiting 20s for some data to be ingested
sleep 20
echo "Initial pod ingested data checks"
./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" app="std-printer" parser="test-parser-1" stream="stdout" "stdout: line 2"'
./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" app="std-printer" parser="test-parser-1" stream="stderr" "stderr: line 2"'
echo ""
kubectl describe pod ${APP_POD_NAME}
kubectl annotate --overwrite pods ${APP_POD_NAME} log.config.scalyr.com/attributes.parser="changed"
kubectl describe pod ${APP_POD_NAME}
# Give agent some time to pick up the annotation change (by default we poll every 30 seconds
# for pod metadata changes, but we use lower value for the tests)
sleep 15
echo ""
echo "Post annotation change data checks"
./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" app="std-printer" parser="changed" stream="stdout" "stdout: line"'
./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" app="std-printer" parser="changed" stream="stderr" "stderr: line"'
echo ""
# Verify Kubernetes Events Monitor events are ingested
./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile="/var/log/scalyr-agent-2/kubernetes_events.log" $parser="k8sEvents" "\"kind\":\"Event\""'
./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile="/var/log/scalyr-agent-2/kubernetes_events.log" $parser="k8sEvents" "\"kind\":\"Pod\""'
./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile="/var/log/scalyr-agent-2/kubernetes_events.log" $parser="k8sEvents" "involvedObject"'
# This event is not always available under Kubernetes >= 1.25
#./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile="/var/log/scalyr-agent-2/kubernetes_events.log" $parser="k8sEvents" "NodeReady"'
agent_status=$(kubectl --namespace=scalyr exec -i ${SCALYR_AGENT_POD_NAME} --container scalyr-agent -- scalyr-agent-2 status -v)
k8s_event_log_files_number=$(echo "$agent_status" | grep -c 'Path /var/log/scalyr-agent-2/kubernetes_events.log: copied')
if [ "$k8s_event_log_files_number" != 1 ]; then
echo "Kubernetes event monitor log has to be handled by the agent 1 time, but it got ${k8s_event_log_files_number}"
exit 1
fi
echo "Wait for BEGIN DELAYED MESSAGE-END DELAYED MESSAGE lines. They should not be splitted."
MINIMUM_RESULTS=10 ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" app="std-printer" parser="test-parser-1" stream="stdout" "BEGIN DELAYED MESSAGE stdout"'
MINIMUM_RESULTS=10 ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" app="std-printer" parser="test-parser-1" stream="stderr" "BEGIN DELAYED MESSAGE stderr"'
echo "Looking for partial lines."
RETRY_ATTEMPTS=1 ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" app="std-printer" parser="test-parser-1" stream="stdout" "BEGIN DELAYED MESSAGE stdout" !"BEGIN DELAYED MESSAGE stdout-END DELAYED MESSAGE stdout"' && echo -e $ERR_MSG && exit 1 || echo -e $SUCCESS_MSG
RETRY_ATTEMPTS=1 ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" app="std-printer" parser="test-parser-1" stream="stderr" "BEGIN DELAYED MESSAGE stderr" !"BEGIN DELAYED MESSAGE stderr-END DELAYED MESSAGE stderr"' && echo -e $ERR_MSG && exit 1 || echo -e $SUCCESS_MSG
echo 'Looking for mixed streams'
RETRY_ATTEMPTS=1 ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" app="std-printer" parser="test-parser-1" stream="stdout" "stderr"' && echo -e $MIXED_STREAMS_ERR_MSG && exit 1 || echo -e $MIXED_STREAMS_SUCCESS_MSG
RETRY_ATTEMPTS=1 ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" app="std-printer" parser="test-parser-1" stream="stderr" "stdout"' && echo -e $MIXED_STREAMS_ERR_MSG && exit 1 || echo -e $MIXED_STREAMS_SUCCESS_MSG
echo "Wait for BEGIN_LONG_MESSAGE .* END_LONG_MESSAGE lines. They should not be splitted."
MINIMUM_RESULTS=10 ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" app="long-message-printer" parser="test-parser-2" stream="stdout" "stdout_BEGIN_LONG_MESSAGE"'
MINIMUM_RESULTS=10 ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" app="long-message-printer" parser="test-parser-2" stream="stderr" "stderr_BEGIN_LONG_MESSAGE"'
echo "Looking for partial lines."
RETRY_ATTEMPTS=1 ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" app="long-message-printer" parser="test-parser-2" stream="stdout" "stdout_BEGIN_LONG_MESSAGE" !(message matches "BEGIN_LONG_MESSAGE.*END_LONG_MESSAGE")' && echo -e $ERR_MSG && exit 1 || echo -e $SUCCESS_MSG
RETRY_ATTEMPTS=1 ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" app="long-message-printer" parser="test-parser-2" stream="stderr" "stderr_BEGIN_LONG_MESSAGE" !(message matches "BEGIN_LONG_MESSAGE.*END_LONG_MESSAGE")' && echo -e $ERR_MSG && exit 1 || echo -e $SUCCESS_MSG
echo 'Looking for mixed streams'
RETRY_ATTEMPTS=1 ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" app="long-message-printer" parser="test-parser-2" stream="stdout" "stderr"' && echo -e $MIXED_STREAMS_ERR_MSG && exit 1 || echo -e $MIXED_STREAMS_SUCCESS_MSG
RETRY_ATTEMPTS=1 ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" app="long-message-printer" parser="test-parser-2" stream="stderr" "stdout"' && echo -e $MIXED_STREAMS_ERR_MSG && exit 1 || echo -e $MIXED_STREAMS_SUCCESS_MSG
echo ""
# We only assert this under newer Kubernetes versions since in the old ones CronJobs were not
# available out of the box
- name: Verify Job and CronJob events are ingested
if: ${{ matrix.k8s_version.run_cronjob_checks }}
timeout-minutes: 5
env:
# Needed for scalyr-tool
scalyr_readlog_token: "${{ secrets.CT_SCALYR_TOKEN_PROD_US_CLOUDTECH_TESTING_READ }}"
SCALYR_AGENT_POD_NAME: "${{ env.SCALYR_AGENT_POD_NAME }}"
K8S_NODE_NAME: "${{ env.K8S_NODE_NAME }}"
run: |
export RETRY_ATTEMPTS="14"
export SLEEP_DELAY="10"
# Wait for at least one hello-printer job to be scheduled and completed.
RETRIES=1;
until echo -n "Waiting for first job to be completed - retry $RETRIES: " && kubectl wait --for=condition=complete job -l app=hello-printer || [ $RETRIES -gt 60 ]; do sleep 1; RETRIES=$(($RETRIES+1)); done
[ $RETRIES -gt 60 ] && echo "Job didn't start in time" && exit 1
# Verify agent is running
echo "Job events checks"
./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile="/var/log/scalyr-agent-2/kubernetes_events.log" $parser="k8sEvents" k8s-kind="Job" watchEventType="ADDED" reason="Completed"'
echo "CronJob events checks"
./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile="/var/log/scalyr-agent-2/kubernetes_events.log" $parser="k8sEvents" k8s-kind="CronJob" involvedObjectKind="CronJob" involvedObjectName="hello" watchEventType="ADDED" reason="SawCompletedJob"'
./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" "Hello from the Kubernetes cluster"'
- name: Check the short lived pods logs are ingested - 404 response handling
if: ${{ matrix.k8s_version.runtime != 'docker' }}
env:
# Needed for scalyr-tool
scalyr_readlog_token: "${{ secrets.CT_SCALYR_TOKEN_PROD_US_CLOUDTECH_TESTING_READ }}"
SCALYR_AGENT_POD_NAME: "${{ env.SCALYR_AGENT_POD_NAME }}"
run: |
./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" message matches "Pod default/short-lived-job-1.* not found in K8s API. Including pod based on SCALYR_K8S_INCLUDE_ALL_CONTAINERS=true."'
./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" message matches "Pod default/short-lived-job-2.* not found in K8s API. Including pod based on SCALYR_K8S_INCLUDE_ALL_CONTAINERS=true."'
./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" message matches "Pod default/short-lived-job-3.* not found in K8s API. Including pod based on SCALYR_K8S_INCLUDE_ALL_CONTAINERS=true."'
- name: Check the short lived pods logs are ingested - log message
env:
# Needed for scalyr-tool
scalyr_readlog_token: "${{ secrets.CT_SCALYR_TOKEN_PROD_US_CLOUDTECH_TESTING_READ }}"
SCALYR_AGENT_POD_NAME: "${{ env.SCALYR_AGENT_POD_NAME }}"
run: |
RETRY_ATTEMPTS=20 ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" "Message from short-lived-container-1"'
RETRY_ATTEMPTS=20 ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" "Message from short-lived-container-2"'
RETRY_ATTEMPTS=20 ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" "Message from short-lived-container-3"'
- name: Install Testing Python Packages
run: |
python3 -m pip install kubernetes tabulate
- name: Verify multiaccount records have been ingested
timeout-minutes: 14
env:
SCALYR_API_KEY_READ_TEAM_1: "${{ secrets.CT_SCALYR_TOKEN_PROD_US_CLOUDTECH_TESTING_READ }}"
SCALYR_API_KEY_READ_TEAM_2: "${{ secrets.SCALYR_API_KEY_READ_2 }}"
SCALYR_API_KEY_READ_TEAM_3: "${{ secrets.SCALYR_API_KEY_READ_3 }}"
SCALYR_API_KEY_READ_TEAM_4: "${{ secrets.SCALYR_API_KEY_READ_4 }}"
SCALYR_API_KEY_READ_TEAM_5: "${{ secrets.SCALYR_API_KEY_READ_5 }}"
SCALYR_API_KEY_READ_TEAM_6: "${{ secrets.SCALYR_API_KEY_READ_6 }}"
SCALYR_API_KEY_READ_TEAM_7: "${{ secrets.SCALYR_API_KEY_READ_7 }}"
SERVER_HOST: "${{ env.SCALYR_AGENT_POD_NAME }}"
ACCOUNT_NAME_1: "[email protected]"
ACCOUNT_NAME_2: "[email protected]"
ACCOUNT_NAME_3: "[email protected]"
ACCOUNT_NAME_4: "[email protected]"
ACCOUNT_NAME_5: "[email protected]"
ACCOUNT_NAME_6: "[email protected]"
ACCOUNT_NAME_7: "[email protected]"
run: |
echo 'Looking for multiple account logs'
./scripts/cicd/verify-multiaccount-records-have-been-ingested.py
k8s_open_metrics_monitor_tests:
name: OpenMetrics Monitor - k8s ${{ inputs.builder_name }} ${{ matrix.k8s_version.version }}-${{ matrix.k8s_version.runtime}}
runs-on: ubuntu-24.04
if: vars.K8S_TESTS_DISABLED != 'true' && github.ref_name != '_release_build' && inputs.builder_name != 'alpine'
timeout-minutes: 15
needs:
- build_image_tarball
strategy:
fail-fast: false
matrix:
k8s_version:
- { "version": "v1.20.15", "driver": "", "runtime": "docker" }
- { "version": "v1.21.10", "driver": "", "runtime": "docker" }
- { "version": "v1.22.7", "driver": "", "runtime": "docker" }
# NOTE: Using containerd runtime in minikube on GHA only works with docker driver
- { "version": "v1.23.17", "driver": "docker", "runtime": "containerd" }
- { "version": "v1.24.14", "driver": "docker", "runtime": "containerd" }
- { "version": "v1.25.10", "driver": "docker", "runtime": "containerd" }
- { "version": "v1.26.5", "driver": "docker", "runtime": "containerd" }
# CRI-O
- { "version": "v1.27.4", "driver": "docker", "runtime": "cri-o" }
steps:
- name: Checkout Repository
uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4
with:
fetch-depth: 0
- name: install python and requirements
uses: ./.github/actions/install_python_and_requirements
with:
python_version: ${{ inputs.python_version }}
- name: Set up QEMU
id: qemu
uses: docker/setup-qemu-action@v3
with:
image: tonistiigi/binfmt:qemu-v6.2.0
platforms: all
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
with:
driver-opts: network=host
- name: Install Scalyr tool
run: |
curl https://raw.githubusercontent.com/scalyr/scalyr-tool/master/scalyr > scalyr
chmod +x scalyr
sudo mv scalyr /usr/local/bin
- name: Setup minikube k8s cluster
uses: ./.github/actions/setup-minikube-cluster/
with:
k8s_version: "${{ matrix.k8s_version.version }}"
minikube_driver: "${{ matrix.k8s_version.driver }}"
container_runtime: "${{ matrix.k8s_version.runtime }}"
github_token: "${{ secrets.GITHUB_TOKEN }}"
- name: Download OCI tarball of the built image.
uses: actions/download-artifact@v4
with:
name: container-image-tarball-k8s-${{ inputs.builder_name }}
path: /tmp/image_tarball
# TODO Investigate, no longer working
#- name: Expose GitHub Runtime To Be Able to Use GHA Cache By Docker.
# uses: crazy-max/ghaction-github-runtime@715c25b40ccc0df9b62bfa8be3ccc57d09dbc4b1
- name: Build Agent k8s Test Docker Image
env:
USE_GHA_CACHE: "1"
CACHE_VERSION: ${{ inputs.cache_version }}
run: |
python3 tests/end_to_end_tests/container_images_test/scripts/build_test_image.py \
--image-builder-name ${{ inputs.builder_name }} \
--base-image ${{ inputs.base_image }} \
--architecture x86_64 \
--image-oci-tarball /tmp/image_tarball/k8s-${{ inputs.builder_name }}.tar \
--result-image-name k8s-image:test \
--image-type k8s
docker image ls
# Needed for containerd or cri-o runtime
if [ "${{ matrix.k8s_version.runtime }}" = "containerd" ] || [ "${{ matrix.k8s_version.runtime }}" = "cri-o" ]; then
docker tag k8s-image:test docker.io/library/k8s-image:test
minikube image load docker.io/library/k8s-image:test
fi
# Here we build the dummy Java app image which exposes JMX metrics via exporter
- name: Build Test Java App Docker Image
run: |
pushd tests/e2e/k8s_om_monitor/java-hello-world
docker build -t java-hello-world .
popd
docker image ls
# Needed for containerd or cri-o runtime
if [ "${{ matrix.k8s_version.runtime }}" = "containerd" ] || [ "${{ matrix.k8s_version.runtime }}" = "cri-o" ]; then
docker tag java-hello-world:latest docker.io/library/java-hello-world:latest
minikube image load docker.io/library/java-hello-world:latest
fi
# Create mock pods and exporters which will be scrapped by the monitor
- name: Create mock pods and exporters
run: |
kubectl create namespace monitoring
# 1. node exporter pod
kubectl apply -f tests/e2e/k8s_om_monitor/node_exporter.yaml
# 2. kube state metrics deployment
kubectl apply -k tests/e2e/k8s_om_monitor/kube-state-metrics/
# 3. Install dummy java app container with jmx exporter side
kubectl apply -f tests/e2e/k8s_om_monitor/java_app_deployment.yaml
sleep 2
kubectl wait --for=condition=ready pod -l app=node-exporter --namespace monitoring ||
(echo "Failed to wait for node-exporter pod to become ready"; kubectl describe pods -l app=node-exporter --namespace monitoring; echo; echo GET PODS:; kubectl get pods -A; echo LOGS:; kubectl logs -l app=node-exporter --namespace monitoring; exit 1)
kubectl wait --for=condition=ready pod -l app=java-hello-world ||
(echo "Failed to wait for java-hello-world pod to become ready"; kubectl describe pods -l app=java-hello-world; echo; echo GET PODS:; kubectl get pods -A; echo LOGS:; kubectl logs -l app=java-hello-world; exit 1)
kubectl get pods -A
- name: Create scalyr-agent-2 daemonset
uses: ./.github/actions/install-k8s-agent/
with:
scalyr_server: "agent.scalyr.com"
scalyr_api_key: "${{ secrets.CT_SCALYR_TOKEN_PROD_US_CLOUDTECH_TESTING_WRITE }}"
scalyr_cluster_name: "${K8S_CLUSTER_NAME}"
main_yaml_path: "tests/e2e/k8s_om_monitor/scalyr-agent-2-daemonset.yaml"
# Monitor is not enabled by default yet since it's still in preview and testing phase so
# we expliticly enable it here
extra_yaml_paths: "tests/e2e/k8s_om_monitor/scalyr-agent-extra-config-configmap.yaml"
- name: Verify data has been ingested
timeout-minutes: 14
env:
# Needed for scalyr-tool
scalyr_readlog_token: "${{ secrets.CT_SCALYR_TOKEN_PROD_US_CLOUDTECH_TESTING_READ }}"
SCALYR_AGENT_POD_NAME: "${{ env.SCALYR_AGENT_POD_NAME }}"
K8S_NODE_NAME: "${{ env.K8S_NODE_NAME }}"
K8S_CLUSTER_NAME: "${{ env.K8S_CLUSTER_NAME }}"
run: |
export RETRY_ATTEMPTS="30"
export SLEEP_DELAY="10"
# Verify agent is running
echo "Agent running checks"
./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile="/var/log/scalyr-agent-2/agent.log" "Starting scalyr agent..."'
# Verify monitor is running
echo "Monitor running checks"
kubectl get pods -A
./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile="/var/log/scalyr-agent-2/agent.log" "Found 3 URL(s) to scrape for node"'
./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile="/var/log/scalyr-agent-2/agent.log" "There are currently 3 dynamic and 2 static open metrics monitors running"'
echo ""
# Kubernetes API metrics (static monitor)
echo "Kubernetes API metrics monitor checks"
./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile contains "openmetrics_monitor-'${K8S_NODE_NAME}'-kubernetes-api-metrics.log" "process_max_fds 1000000 k8s-cluster=\"'${K8S_CLUSTER_NAME}'\" k8s-node=\"'${K8S_NODE_NAME}'\""'
./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile contains "openmetrics_monitor-'${K8S_NODE_NAME}'-kubernetes-api-metrics.log" "process_open_fds "'
# Kubernetes API cAdvisor metrics (static monitor)
echo "Kubernetes API cAdvisor metrics monitor checks"
./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile contains "openmetrics_monitor-'${K8S_NODE_NAME}'-kubernetes-api-cadvisor-metrics" "machine_cpu_cores '$(nproc --all)'"'
MINIMUM_RESULTS=2 ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile contains "openmetrics_monitor-'${K8S_NODE_NAME}'-kubernetes-api-cadvisor-metrics" "container_cpu_load_average_10s "'
# Verify locally calculated rate metrics
./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile contains "openmetrics_monitor-'${K8S_NODE_NAME}'-kubernetes-api-cadvisor-metrics" "container_cpu_usage_seconds_total_rate "'
./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile contains "openmetrics_monitor-'${K8S_NODE_NAME}'-kubernetes-api-cadvisor-metrics" "container_network_receive_bytes_total_rate "'
./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile contains "openmetrics_monitor-'${K8S_NODE_NAME}'-kubernetes-api-cadvisor-metrics" "container_network_transmit_bytes_total_rate "'
# 2. Verify node exporter metrics
echo "Node exporter metrics monitor checks"
./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile contains "openmetrics_monitor-'${K8S_NODE_NAME}'-node-exporter-" "process_max_fds "'
./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile contains "openmetrics_monitor-'${K8S_NODE_NAME}'-node-exporter-" "process_open_fds "'
./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile contains "openmetrics_monitor-'${K8S_NODE_NAME}'-node-exporter-" "node_vmstat_pswpin "'
./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile contains "openmetrics_monitor-'${K8S_NODE_NAME}'-node-exporter-" "node_vmstat_pswpout "'
# Verify locally calculated rate metrics
./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile contains "openmetrics_monitor-'${K8S_NODE_NAME}'-node-exporter-" "node_cpu_seconds_total_rate "'
./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile contains "openmetrics_monitor-'${K8S_NODE_NAME}'-node-exporter-" "node_network_transmit_bytes_total_rate "'
./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile contains "openmetrics_monitor-'${K8S_NODE_NAME}'-node-exporter-" "node_network_receive_bytes_total_rate "'
./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile contains "openmetrics_monitor-'${K8S_NODE_NAME}'-node-exporter-" "node_disk_read_bytes_total_rate "'
./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile contains "openmetrics_monitor-'${K8S_NODE_NAME}'-node-exporter-" "node_disk_written_bytes_total_rate "'
# 3. Verify kube state event metrics
echo "Kube state events metrics monitor checks"
./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile contains "openmetrics_monitor-'${K8S_NODE_NAME}'" $logfile contains "kube-state-metrics" "kube_storageclass_labels 1 k8s-cluster=\"'${K8S_CLUSTER_NAME}'\" k8s-node=\"'${K8S_NODE_NAME}'\" storageclass=\"standard\""'
./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile contains "openmetrics_monitor-'${K8S_NODE_NAME}'" $logfile contains "kube-state-metrics" "kube_secret_type 1 k8s-cluster=\"'${K8S_CLUSTER_NAME}'\" k8s-node=\"'${K8S_NODE_NAME}'\" namespace=\"scalyr\" secret=\"scalyr-api-key\" type=\"Opaque\""'
# 4. Verify java app JMX metrics
echo "Java JMX metrics events metrics monitor checks"
./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile contains "openmetrics_monitor-'${K8S_NODE_NAME}'" $logfile contains "java-hello-world" "jmx_scrape_error 0.0 app=\"java-hello-world\" app_instance=\"java-hello-world-1\" attribute1=\"value1\" k8s-cluster=\"'${K8S_CLUSTER_NAME}'\" k8s-node=\"'${K8S_NODE_NAME}'\""'
./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile contains "openmetrics_monitor-'${K8S_NODE_NAME}'" $logfile contains "java-hello-world" "jmx_scrape_cached_beans 0.0 app=\"java-hello-world\" app_instance=\"java-hello-world-1\" attribute1=\"value1\" k8s-cluster=\"'${K8S_CLUSTER_NAME}'\" k8s-node=\"'${K8S_NODE_NAME}'\""'
./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile contains "openmetrics_monitor-'${K8S_NODE_NAME}'" $logfile contains "java-hello-world" "jvm_info 1.0 app=\"java-hello-world\" app_instance=\"java-hello-world-1\" attribute1=\"value1\" k8s-cluster=\"'${K8S_CLUSTER_NAME}'\" k8s-node=\"'${K8S_NODE_NAME}'\" runtime="'
# Client side calculated per second rate metrics
# NOTE: There is no easy way to assert on the actual rate metric value here (we have unit tests for that)
# so we just assert that the metric is present.
./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile contains "openmetrics_monitor-'${K8S_NODE_NAME}'" $logfile contains "java-hello-world" "jvm_threads_started_total_rate" "app=\"java-hello-world\" app_instance=\"java-hello-world-1\" attribute1=\"value1\" k8s-cluster=\"'${K8S_CLUSTER_NAME}'\" k8s-node=\"'${K8S_NODE_NAME}'\""'
./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile contains "openmetrics_monitor-'${K8S_NODE_NAME}'" $logfile contains "java-hello-world" "jvm_memory_pool_allocated_bytes_total_rate" "app=\"java-hello-world\" app_instance=\"java-hello-world-1\" attribute1=\"value1\" k8s-cluster=\"'${K8S_CLUSTER_NAME}'\" k8s-node=\"'${K8S_NODE_NAME}'\""'
./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile contains "openmetrics_monitor-'${K8S_NODE_NAME}'" $logfile contains "java-hello-world" "process_cpu_seconds_total_rate" "app=\"java-hello-world\" app_instance=\"java-hello-world-1\" attribute1=\"value1\" k8s-cluster=\"'${K8S_CLUSTER_NAME}'\" k8s-node=\"'${K8S_NODE_NAME}'\""'
- name: Notify Slack on Failure
if: ${{ failure() && github.ref_name == 'master' }}
uses: act10ns/slack@ed1309ab9862e57e9e583e51c7889486b9a00b0f # v2.0.0
env:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
with:
status: ${{ job.status }}
steps: ${{ toJson(steps) }}
channel: '#eng-dataset-cloud-tech'
# TODO Make decision on whether this is to be replaced or simply removed
# The scalyr-agent-ci-unittest image dependency is problematic because it uses a centos image
# Ref: .circleci/docker_unified_smoke_unit/Dockerfile.unittest
# k8s-smoketest:
# runs-on: ${{ vars.RUNNER }}
# needs:
# - build_image_tarball
# name: K8s Smoketest - ${{ inputs.builder_name }}
# if: vars.K8S_TESTS_DISABLED != 'true' && github.ref_name != '_release_build' && (github.event_name != 'pull_request' || ! github.event.pull_request.draft)
# steps:
# - name: Checkout Repository
# uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4
#
# - name: Install python and requirements
# uses: ./.github/actions/install_python_and_requirements
# with:
# python_version: ${{ inputs.python_version }}
#
# - name: Set up QEMU
# id: qemu
# uses: docker/setup-qemu-action@v3
# with:
# image: tonistiigi/binfmt:qemu-v6.2.0
# platforms: all
#
# - name: Login to DockerHub
# uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3
# with:
# username: ${{ secrets.PULL_REGISTRY_USERNAME }}
# password: ${{ secrets.PULL_REGISTRY_PASSWORD }}
#
# - name: Set up Docker Context for Buildx
# id: buildx-context
# run: |
# docker context create builders || echo Already exists
#
# - name: Set up Docker Buildx
# uses: docker/setup-buildx-action@v3
# with:
# driver-opts: network=host
# endpoint: builders
#
# - name: Setup minikube k8s cluster
# uses: ./.github/actions/setup-minikube-cluster/
# with:
# k8s_version: v1.22.0
# minikube_driver: ""
# container_runtime: "docker"
# github_token: "${{ secrets.GITHUB_TOKEN }}"
#
# - name: Download OCI tarball of the built image.
# uses: actions/download-artifact@v4
# with:
# name: container-image-tarball-k8s-${{ inputs.builder_name }}
# path: /tmp/image_tarball
#
# - name: Build test image
# env:
# USE_GHA_CACHE: "1"
# CACHE_VERSION: ${{ inputs.cache_version }}
# run: |
# python3 tests/end_to_end_tests/container_images_test/scripts/build_test_image.py \
# --image-builder-name ${{ inputs.builder_name }} \
# --base-image ${{ inputs.base_image }} \
# --architecture x86_64 \
# --image-oci-tarball /tmp/image_tarball/k8s-${{ inputs.builder_name }}.tar \
# --result-image-name k8s-image:test \
# --image-type k8s
#
# - name: Run Tests (with coverage)
# env:
# CIRCLE_BUILD_NUM: ${{ github.run_number }}
# SCALYR_SERVER: https://agent.scalyr.com
# SCALYR_API_KEY: ${{ secrets.CT_SCALYR_TOKEN_PROD_US_CLOUDTECH_TESTING_WRITE }}
# READ_API_KEY: ${{ secrets.CT_SCALYR_TOKEN_PROD_US_CLOUDTECH_TESTING_READ }}
# run: |
# source ./.circleci/smoketest_k8s.sh scalyr/scalyr-agent-ci-unittest:4 300 no_delete_existing_k8s_objs k8s-image:test
#
# - name: Upload test results
# uses: actions/upload-artifact@v4
# with:
# name: k8s-smoketest
# path: |
# .coverage
# if: ${{ success() || failure() }}
# TODO Make decision on whether this is to be replaced or simply removed
# The scalyr-agent-ci-unittest image dependency is problematic because it uses a centos image
# Ref: .circleci/docker_unified_smoke_unit/Dockerfile.unittest
# docker-smoketest:
# name: Docker Smoketest - ${{ inputs.builder_name }} ${{ matrix.variant.image_type }}
# if: github.ref_name != '_release_build'
# needs:
# - build_image_tarball
# runs-on: ${{ vars.RUNNER }}
# strategy:
# fail-fast: false
# matrix:
# variant:
# - image_type: docker-json
# timeout: 300
# - image_type: docker-syslog
# timeout: 300
# - image_type: docker-api
# timeout: 300
# steps:
# - name: Checkout Repository
# uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4
#
# - name: install python and requirements
# uses: ./.github/actions/install_python_and_requirements
# with:
# python_version: ${{ inputs.python_version }}
#
# - name: Set up QEMU
# id: qemu
# uses: docker/setup-qemu-action@v3
# with:
# image: tonistiigi/binfmt:qemu-v6.2.0
# platforms: all
#
# - name: Login to DockerHub
# uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3
# with:
# username: ${{ secrets.PULL_REGISTRY_USERNAME }}
# password: ${{ secrets.PULL_REGISTRY_PASSWORD }}
#
# - name: Login to an internal build registry
# if: vars.INTERNAL_BUILD_REGISTRY
# uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3
# continue-on-error: true
# with:
# registry: ${{ vars.INTERNAL_BUILD_REGISTRY }}
# username: ${{ secrets.INTERNAL_BUILD_REGISTRY_USERNAME }}
# password: ${{ secrets.INTERNAL_BUILD_REGISTRY_PASSWORD }}
#
# # Context need to be created for a self-hosted runner
# - name: Set up Docker Context for Buildx
# id: buildx-context
# run: |
# docker context create builders || echo Already exists
#
# - name: Set up Docker Buildx
# uses: docker/setup-buildx-action@v3
# with:
# driver-opts: network=host
# endpoint: builders
#
# # TODO Investigate, no longer working
# #- name: Expose GitHub Runtime To Be Able to Use GHA Cache By Docker.
# # uses: crazy-max/ghaction-github-runtime@715c25b40ccc0df9b62bfa8be3ccc57d09dbc4b1
#
# - name: Download OCI tarball of the built image.
# uses: actions/download-artifact@v4
# with:
# name: container-image-tarball-${{ matrix.variant.image_type }}-${{ inputs.builder_name }}
# path: /tmp/image_tarball
#
# - name: Build test image
# env:
# USE_GHA_CACHE: "1"
# CACHE_VERSION: ${{ inputs.cache_version }}
# run: |
# python3 tests/end_to_end_tests/container_images_test/scripts/build_test_image.py \
# --image-builder-name ${{ inputs.builder_name }} \
# --base-image ${{ inputs.base_image }} \
# --architecture x86_64 \
# --image-oci-tarball /tmp/image_tarball/${{ matrix.variant.image_type }}-${{ inputs.builder_name }}.tar \
# --result-image-name ${{ matrix.variant.image_type }}-${{ inputs.builder_name }}:test \
# --image-type ${{ matrix.variant.image_type }}
#
# - name: Run Tests (with coverage)
# env:
# CIRCLE_BUILD_NUM: ${{ github.run_number }}
# SCALYR_SERVER: https://agent.scalyr.com
# SCALYR_API_KEY: ${{ secrets.CT_SCALYR_TOKEN_PROD_US_CLOUDTECH_TESTING_WRITE }}
# READ_API_KEY: ${{ secrets.CT_SCALYR_TOKEN_PROD_US_CLOUDTECH_TESTING_READ }}
# run: |
# source ./.circleci/smoketest_docker.sh scalyr/scalyr-agent-ci-unittest:4 ${{ matrix.variant.image_type }} ${{ matrix.variant.timeout }} ${{ matrix.variant.image_type }}-${{ inputs.builder_name }}:test
#
# - name: Upload test results
# uses: actions/upload-artifact@v4
# with:
# name: docker-smoketest-${{ matrix.variant.image_type }}-${{ inputs.builder_name }}
# path: |
# .coverage
# if: ${{ success() || failure() }}
#
# - name: Notify Slack on Failure
# if: ${{ failure() && github.ref_name == 'master' }}
# uses: act10ns/slack@ed1309ab9862e57e9e583e51c7889486b9a00b0f # v2.0.0
# env:
# SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
# with:
# status: ${{ job.status }}
# steps: ${{ toJson(steps) }}
# channel: '#eng-dataset-cloud-tech'
publish_images:
name: Publish image ${{ matrix.variant.image_type }}-${{ inputs.builder_name }}
runs-on: ubuntu-24.04
if: github.ref_type == 'tag' || github.ref_name == 'master' || github.head_ref == 'release' || github.event_name == 'pull_request'
needs:
# k8s_kubernetes_monitor_tests, k8s_open_metrics_monitor_tests, k8s-smoketest checks are not needed before pushing a test version to dockerhub.
# They need to pass to allow PR merge.
- build_image_tarball
- docker-smoketest
strategy:
matrix:
variant:
- { "image_type": "k8s" }
- { "image_type": "docker-json" }
- { "image_type": "docker-syslog" }
- { "image_type": "docker-api" }
steps:
- name: Checkout Repository
uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4
- name: install python and requirements
uses: ./.github/actions/install_python_and_requirements
with:
python_version: ${{ inputs.python_version }}
- name: Download OCI tarball of the built image.
uses: actions/download-artifact@v4
with:
name: container-image-tarball-${{ matrix.variant.image_type }}-${{ inputs.builder_name }}
path: /tmp/image_tarball
- name: Verify and Create Tags
id: verify
shell: bash
run: |
AGENT_VERSION="$(cat VERSION | tr -d '\n')"
if [ "${{ github.ref_type }}" = "tag" ]; then
if [ "${{ github.ref_name }}" = "v${AGENT_VERSION}" ]; then
echo "tags=$AGENT_VERSION,latest" >> "${GITHUB_OUTPUT}"
else
echo "tags=empty" >> "${GITHUB_OUTPUT}"
fi
elif [ "${{ github.ref_name }}" = "master" ]; then
echo "tags=${{ github.sha }}" >> "${GITHUB_OUTPUT}"
elif [ "${{ github.head_ref }}" = "release" ]; then
echo "tags=latest-release,pr-${{ github.event.number}}-${{ github.event.pull_request.commits}}-${{ github.sha }},\
$AGENT_VERSION,latest" >> "${GITHUB_OUTPUT}"
elif [ "${{ github.event_name }}" = "pull_request" ]; then
echo "tags=latest-pr,pr-${{ github.event.number}}-${{ github.event.pull_request.commits}}-${{ github.sha }}" >> "${GITHUB_OUTPUT}"
else
echo "tags=empty" >> "${GITHUB_OUTPUT}"
fi
- name: Set up skopeo
uses: warjiang/setup-skopeo@main
with:
version: latest
- name: Publish image
if: steps.verify.outputs.tags != 'empty'
run: |
python3 build_package_new.py \
image \
${{ inputs.builder_name }} \
--base-image ${{ inputs.base_image }} \
publish \
--registry docker.io \
--no-verify-tls \
--name-prefix "${{ secrets.PUBLISH_REGISTRY_USERNAME }}" \
--tags "${{ steps.verify.outputs.tags }}" \
--from-oci-layout /tmp/image_tarball/${{ matrix.variant.image_type }}-${{ inputs.builder_name }}.tar \
--registry-username "${{ secrets.PUBLISH_REGISTRY_USERNAME }}" \
--registry-password "${{ secrets.PUBLISH_REGISTRY_PASSWORD }}" \
--image-type ${{ matrix.variant.image_type }}