L7 Policy Perf Test (l7-perf) #104

Workflow file for this run

.github/workflows/l7-perf.yaml at 149b3a0

	name: L7 Policy Perf Test (l7-perf)

	on:
	schedule:
	- cron: "27 0 * * 1-5"

	workflow_dispatch:
	inputs:
	PR-number:
	description: "Pull request number."
	required: true
	context-ref:
	description: "Context in which the workflow runs. If PR is from a fork, will be the PR target branch (general case). If PR is NOT from a fork, will be the PR branch itself (this allows committers to test changes to workflows directly from PRs)."
	required: true
	SHA:
	description: "SHA under test (head of the PR branch)."
	required: true
	num-test-nodes:
	description: "Number of test nodes to create"
	required: false
	default: 1
	type: number
	num-client-nodes:
	description: "Number of client nodes to create/use for testing"
	required: false
	default: 1
	type: number
	extra-args:
	description: "[JSON object] Arbitrary arguments passed from the trigger comment via regex capture group. Parse with 'fromJson(inputs.extra-args).argName' in workflow."
	required: false
	default: '{}'

	permissions:
	# To be able to retrieve artifacts information
	actions: read
	# To be able to access the repository with actions/checkout
	contents: read
	# To be able to request the JWT from GitHub's OIDC provider
	id-token: write
	# To allow retrieving information from the PR API
	pull-requests: read
	# To be able to set commit status
	statuses: write

	concurrency:
	# Structure:
	# - Workflow name
	# - Event type
	# - A unique identifier depending on event type:
	# - schedule: SHA
	# - workflow_dispatch: PR number
	#
	# This structure ensures a unique concurrency group name is generated for each
	# type of testing, such that re-runs will cancel the previous run.
	group: \|
	${{ github.workflow }}
	${{ github.event_name }}
	${{
	(github.event_name == 'schedule' && github.sha) \|\|
	(github.event_name == 'workflow_dispatch' && github.event.inputs.PR-number)
	}}
	cancel-in-progress: true

	env:
	# renovate: datasource=github-releases depName=eksctl-io/eksctl
	eksctl_version: v0.224.0
	# renovate: datasource=github-releases depName=kubernetes/kubernetes
	kubectl_version: v1.35.3
	# renovate: datasource=docker depName=google/cloud-sdk
	gcloud_version: 561.0.0

	test_name: l7-perf
	cluster_name: ${{ github.run_id }}-${{ github.run_attempt }}
	nighthawk_docker_image: docker.io/envoyproxy/nighthawk-dev:ee3f687f3485cf72f0c4262def369dfd89ad7ed1@sha256:b13687c0ac49025777ae8ed669b28fbf03626830d959e7452ec408d54ad8ffd1

	jobs:
	echo-inputs:
	if: ${{ github.event_name == 'workflow_dispatch' }}
	name: Echo Workflow Dispatch Inputs
	runs-on: ubuntu-24.04
	steps:
	- name: Echo Workflow Dispatch Inputs
	run: \|
	echo '${{ tojson(inputs) }}'

	commit-status-start:
	name: Commit Status Start
	runs-on: ubuntu-24.04
	steps:
	- name: Set initial commit status
	uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1
	with:
	sha: ${{ inputs.SHA \|\| github.sha }}

	install-and-perftest:
	permissions:
	# To be able to trigger eks-cluster-pool-manager.yaml workflow
	actions: write
	# To be able to access the repository with actions/checkout
	contents: read
	# To be able to request the JWT from GitHub's OIDC provider
	id-token: write
	# To allow retrieving information from the PR API
	pull-requests: read
	# To be able to set commit status
	statuses: write
	runs-on: ubuntu-24.04
	name: Install and Perf Test
	timeout-minutes: 60
	strategy:
	fail-fast: false
	matrix:
	test_type:
	- benchmark
	steps:
	- name: Set commit status to pending
	uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1
	with:
	sha: ${{ inputs.SHA \|\| github.sha }}
	status: pending

	- name: Checkout context ref (trusted)
	uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
	with:
	ref: ${{ inputs.context-ref \|\| github.sha }}
	persist-credentials: false

	- name: Set Environment Variables
	uses: ./.github/actions/set-env-variables

	- name: Set up job variables
	id: vars
	run: \|
	if [ "${{ github.event_name }}" == "workflow_dispatch" ] ; then
	SHA="${{ inputs.SHA }}"
	else
	SHA="${{ github.sha }}"
	fi

	# Run test against specific commit SHA if provided
	if [[ '${{ inputs.extra-args }}' =~ sha=([a-f0-9]+) ]]; then
	SHA=${BASH_REMATCH[1]}
	fi

	# Retrieve the desired version from the arguments.
	if [[ '${{ inputs.extra-args }}' =~ version=([-+.0-9a-z]+) ]]; then
	VERSION=${BASH_REMATCH[1]}
	else
	VERSION=""
	fi

	# * The SHA under test will have its helm chart checked out at the following
	# path right before the step where Cilium is installed.
	CILIUM_INSTALL_DEFAULTS="--wait=false \
	--set=hubble.enabled=true \
	--set=pprof.enabled=true \
	--set=prometheus.enabled=true \
	--set=cluster.name=${{ env.cluster_name }} \
	--set=kubeProxyReplacement=true \
	--set=l7Proxy=true \
	--set envoyConfig.enabled=true \
	--set=eni.enabled=true \
	--set=ipam.mode=eni \
	--set=eni.awsEnablePrefixDelegation=true"

	# only add SHA to the image tags if VERSION is not set
	if [ -z "${VERSION}" ]; then
	CILIUM_INSTALL_DEFAULTS="--chart-directory=untrusted/install/kubernetes/cilium \
	${CILIUM_INSTALL_DEFAULTS} \
	--set=image.override=quay.io/${{ env.QUAY_ORGANIZATION_DEV }}/cilium-ci:${SHA} \
	--set=operator.image.override=quay.io/${{ env.QUAY_ORGANIZATION_DEV }}/operator-aws-ci:${SHA}"
	else
	CILIUM_INSTALL_DEFAULTS="--version ${VERSION} ${CILIUM_INSTALL_DEFAULTS}"
	fi

	OWNER="${{ github.ref_name }}"
	OWNER="${OWNER//[.\/]/-}"

	if [ "${{ github.event_name }}" == "workflow_dispatch" ] ; then
	NUM_TEST_NODES="${{ inputs.num-test-nodes }}"
	NUM_CLIENT_NODES="${{ inputs.num-client-nodes }}"
	else
	NUM_TEST_NODES="1"
	NUM_CLIENT_NODES="1"
	fi

	# m5n.xlarge instances support up to 25Gbps burst bandwidth.
	NODE_INSTANCE_TYPE="m5n.xlarge"

	TEST_NAME="${{ env.test_name }}-${{ matrix.test_type }}-${NUM_TEST_NODES}"
	CLUSTER_NAME="${TEST_NAME}-${{ env.cluster_name }}"
	NIGHTHAWK_DOCKER_IMAGE="${{ env.nighthawk_docker_image }}"

	eks_version_and_region=$(yq '.include \| sort_by(.version) \| reverse \| .[0] \| "\(.version),\(.region)"' .github/actions/eks/k8s-versions.yaml)
	EKS_VERSION=$(echo $eks_version_and_region \| cut -d',' -f1)
	EKS_REGION=$(echo $eks_version_and_region \| cut -d',' -f2)

	echo sha=${SHA} >> $GITHUB_OUTPUT
	echo cilium_version=${VERSION} >> $GITHUB_OUTPUT
	echo cilium_install_defaults=${CILIUM_INSTALL_DEFAULTS} >> $GITHUB_OUTPUT
	echo owner=${OWNER} >> $GITHUB_OUTPUT
	echo test_name=${TEST_NAME} >> $GITHUB_OUTPUT
	echo cluster_name=${CLUSTER_NAME} >> $GITHUB_OUTPUT
	echo nighthawk_docker_image=${NIGHTHAWK_DOCKER_IMAGE} >> $GITHUB_OUTPUT
	echo node_instance_type=${NODE_INSTANCE_TYPE} >> $GITHUB_OUTPUT
	echo eks_version=${EKS_VERSION} >> $GITHUB_OUTPUT
	echo eks_region=${EKS_REGION} >> $GITHUB_OUTPUT
	echo eks_zone_1=${EKS_REGION}b >> $GITHUB_OUTPUT
	echo eks_zone_2=${EKS_REGION}c >> $GITHUB_OUTPUT

	echo num_test_nodes=${NUM_TEST_NODES} >> $GITHUB_OUTPUT
	echo num_client_nodes=${NUM_CLIENT_NODES} >> $GITHUB_OUTPUT

	# Warning: since this is a privileged workflow, subsequent workflow job
	# steps must take care not to execute untrusted code.
	- name: Checkout context ref (NOT TRUSTED)
	if: ${{ steps.vars.outputs.cilium_version == '' }} # We don't need to checkout the PR if released version installation is requested.
	uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
	with:
	ref: ${{ steps.vars.outputs.SHA }}
	persist-credentials: false
	path: untrusted
	sparse-checkout: \|
	install/kubernetes/cilium

	- name: Wait for Cilium images
	if: ${{ steps.vars.outputs.cilium_version == '' }}
	uses: ./.github/actions/wait-for-images
	with:
	SHA: ${{ steps.vars.outputs.SHA }}
	images: cilium-ci operator-aws-ci cilium-cli-ci

	- name: Wait for Nighthawk Benchmarking framework image
	shell: bash
	run: \|
	if ! docker manifest inspect ${{ env.nighthawk_docker_image }} ; then
	echo "FATAL: nighthawk image is not available, exiting"
	exit 1
	fi

	- name: Install Go
	uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0
	with:
	# renovate: datasource=golang-version depName=go
	go-version: 1.26.1

	- name: Setup gcloud credentials
	uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3.0.0
	with:
	workload_identity_provider: ${{ secrets.GCP_PERF_WORKLOAD_IDENTITY_PROVIDER }}
	service_account: ${{ secrets.GCP_PERF_SA }}
	create_credentials_file: true
	export_environment_variables: true

	- name: Setup gcloud CLI
	uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db # v3.0.1
	with:
	project_id: ${{ secrets.GCP_PERF_PROJECT_ID }}
	version: ${{ env.gcloud_version }}

	- name: Install kubectl
	run: \|
	curl -sLO "https://dl.k8s.io/release/${{ env.kubectl_version }}/bin/linux/amd64/kubectl"
	curl -sLO "https://dl.k8s.io/${{ env.kubectl_version }}/bin/linux/amd64/kubectl.sha256"
	echo "$(cat kubectl.sha256) kubectl" \| sha256sum --check
	sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
	kubectl version --client

	- name: Install eksctl CLI
	run: \|
	curl -LO "https://github.com/eksctl-io/eksctl/releases/download/${{ env.eksctl_version }}/eksctl_$(uname -s)_amd64.tar.gz"
	sudo tar -xzvf "eksctl_$(uname -s)_amd64.tar.gz" -C /usr/bin
	rm "eksctl_$(uname -s)_amd64.tar.gz"

	- name: Set up AWS CLI credentials
	uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 # v6.0.0
	with:
	role-to-assume: ${{ secrets.AWS_PR_ASSUME_ROLE }}
	aws-region: ${{ steps.vars.outputs.eks_region }}

	- name: Display version info of installed tools
	run: \|
	echo "--- go ---"
	go version
	echo "--- kubectl ---"
	kubectl version --client
	echo "--- eksctl ---"
	eksctl version
	echo "--- gcloud ---"
	gcloud version

	- name: Clone ClusterLoader2
	uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
	with:
	repository: kubernetes/perf-tests
	# Avoid using renovate to update this dependency because: (1)
	# perf-tests does not tag or release, so renovate will pull
	# all updates to the default branch and (2) continually
	# updating CL2 may impact the stability of the scale test
	# results.
	ref: d51da38a445d653c6f7cd039728d313cb31290b9
	persist-credentials: false
	sparse-checkout: clusterloader2
	path: perf-tests

	- name: Create EKS cluster
	id: deploy-cluster
	uses: ./.github/actions/setup-eks-cluster
	with:
	cluster_name: ${{ steps.vars.outputs.cluster_name }}
	region: ${{ steps.vars.outputs.eks_region }}
	zones: "${{ steps.vars.outputs.eks_zone_1 }} ${{ steps.vars.outputs.eks_zone_2 }}"
	owner: "${{ steps.vars.outputs.owner }}"
	version: "${{ steps.vars.outputs.eks_version }}"
	addons: "coredns"

	- name: Generate cilium-cli kubeconfig
	id: gen-kubeconfig
	uses: ./.github/actions/get-cloud-kubeconfig
	with:
	kubeconfig: "~/.kube/config"

	- name: Install Cilium CLI
	uses: cilium/cilium-cli@3ce83aa3b4ea3ce6d97023c4d83c22b73a4ed3b4 # v0.19.2
	with:
	skip-build: ${{ env.CILIUM_CLI_SKIP_BUILD }}
	image-repo: ${{ env.CILIUM_CLI_IMAGE_REPO }}
	image-tag: ${{ inputs.SHA \|\| github.sha }}
	repository: ${{ env.CILIUM_CLI_RELEASE_REPO }}
	release-version: ${{ env.CILIUM_CLI_VERSION }}
	kubeconfig: ${{ steps.gen-kubeconfig.outputs.kubeconfig_path }}

	- name: Install Cilium
	id: install-cilium
	run: \|
	K8S_API_SRV_ADDR=$(kubectl config view -o jsonpath='{.clusters[0].cluster.server}' \| sed 's\|https://\|\|')
	echo "Retrieved Kubernetes API Server address: '${K8S_API_SRV_ADDR}'"

	# Retrieve the CIDR associated with the availability zone where the cluster
	# is deployed, and configure it as native routing CIDR. Otherwise, Cilium
	# would default to the VPC CIDR, but we want to masquerade traffic towards
	# the external destination (which is located in the same VPC, but different
	# availability zone), to reflect the most common type of real deployments.
	NATIVE_CIDR_BLOCK=$(aws ec2 describe-subnets --region ${{ steps.vars.outputs.eks_region }} \
	--filters Name=availability-zone,Values=${{ steps.vars.outputs.eks_zone_1 }} \
	Name=tag:alpha.eksctl.io/cluster-name,Values=${{ steps.vars.outputs.cluster_name }} \
	Name=map-public-ip-on-launch,Values=false \
	--query 'Subnets[*].CidrBlock' --output text)
	echo "Retrieved native routing CIDR: '${NATIVE_CIDR_BLOCK}'"

	cilium install ${{ steps.vars.outputs.cilium_install_defaults }} \
	--set=k8sServiceHost="${K8S_API_SRV_ADDR}" --set=k8sServicePort=443 \
	--set=ipv4NativeRoutingCIDR="${NATIVE_CIDR_BLOCK}"

	- name: Delete context ref
	run: \|
	rm -rf untrusted/

	# This needs to be performed in a different step, because nodeGroups are not
	# supported during cluster creation in a cluster without VPC CNI. Cilium is
	# also required to be already installed for the step to complete successfully.
	- name: Create EKS nodegroups
	uses: ./.github/actions/setup-eks-nodegroup
	with:
	cluster_name: ${{ steps.vars.outputs.cluster_name }}
	region: ${{ steps.vars.outputs.eks_region }}
	owner: "${{ steps.vars.outputs.owner }}"
	version: "${{ steps.vars.outputs.eks_version }}"
	timeout: "15m"
	node_groups_override: \|
	managedNodeGroups:
	- name: ng-amd64-client
	instanceTypes:
	- ${{ steps.vars.outputs.node_instance_type }}
	availabilityZones:
	- ${{ steps.vars.outputs.eks_zone_1 }}
	desiredCapacity: ${{ steps.vars.outputs.num_client_nodes }}
	spot: false
	privateNetworking: true
	volumeType: "gp3"
	volumeSize: 20
	maxPodsPerNode: 110
	taints:
	- key: "node.cilium.io/agent-not-ready"
	value: "true"
	effect: "NoExecute"
	labels:
	role.scaffolding/http-perf-client: "true"

	- name: ng-amd64-server
	instanceTypes:
	- ${{ steps.vars.outputs.node_instance_type }}
	availabilityZones:
	- ${{ steps.vars.outputs.eks_zone_1 }}
	desiredCapacity: ${{ steps.vars.outputs.num_test_nodes }}
	spot: false
	privateNetworking: true
	volumeType: "gp3"
	volumeSize: 20
	maxPodsPerNode: 110
	taints:
	- key: "node.cilium.io/agent-not-ready"
	value: "true"
	effect: "NoExecute"
	labels:
	role.scaffolding/http-perf-server: "true"

	- name: ng-amd64-monitoring
	instanceTypes:
	- ${{ steps.vars.outputs.node_instance_type }}
	availabilityZones:
	- ${{ steps.vars.outputs.eks_zone_1 }}
	desiredCapacity: 1
	spot: false
	privateNetworking: true
	volumeType: "gp3"
	volumeSize: 20
	maxPodsPerNode: 110
	taints:
	- key: "node.cilium.io/agent-not-ready"
	value: "true"
	effect: "NoExecute"
	labels:
	role.scaffolding/monitoring: "true"

	- name: Wait for Cilium status to be ready
	run: \|
	cilium status --wait --interactive=false

	- name: Run CL2
	id: run-cl2
	working-directory: ./perf-tests/clusterloader2
	shell: bash
	timeout-minutes: 40
	env:
	CL2_PROMETHEUS_PVC_ENABLED: "false"
	CL2_ENABLE_PVS: "false"
	CL2_PROMETHEUS_SCRAPE_CILIUM_OPERATOR: "true"
	CL2_PROMETHEUS_SCRAPE_CILIUM_AGENT: "true"
	CL2_PROMETHEUS_MEMORY_SCALE_FACTOR: "2.0"
	CL2_PROMETHEUS_SCRAPE_CILIUM_AGENT_INTERVAL: "10s"
	CL2_PROMETHEUS_NODE_SELECTOR: 'role.scaffolding/monitoring: "true"'

	CL2_PIN_WORKLOADS_TO_LABELLED_NODES: "true"
	CL2_NUM_TEST_NAMESPACES: 1
	CL2_NUM_TEST_DEPLOYMENTS_PER_NAMESPACE: "${{ steps.vars.outputs.num_test_nodes }}"
	CL2_NUM_POD_INSTANCES_PER_DEPLOYMENT: 1
	CL2_NUM_CLIENT_INSTANCES_PER_DEPLOYMENT: 5
	CL2_NUM_RULES_PER_NETWORK_POLICY: 1000

	CL2_NIGHTHAWK_DOCKER_IMAGE: "${{ steps.vars.outputs.nighthawk_docker_image }}"

	# Set to false for testing the workflow.
	CL2_ENABLE_VIOLATIONS: "true"
	run: \|
	echo "CL2-related environment variables"
	printenv \| grep CL2_

	mkdir ./report
	go run ./cmd/clusterloader.go \
	-v=2 \
	--testconfig=../../.github/actions/cl2-modules/l7/config.yaml \
	--prometheus-additional-monitors-path=../../.github/actions/cl2-modules/l7/monitors \
	--provider=aws \
	--enable-exec-service=false \
	--enable-prometheus-server \
	--enable-pushgateway \
	--prometheus-scrape-kubelets \
	--tear-down-prometheus-server=false \
	--report-dir=./report \
	--experimental-prometheus-snapshot-to-report-dir=true \
	--kubeconfig=$HOME/.kube/config \
	--testoverrides=./testing/prometheus/not-scrape-kube-proxy.yaml \
	2>&1 \| tee cl2-output.txt

	# The cilium-cli creates files owned by the root user when run as a container.
	sudo chmod --recursive +r ./report

	- name: Get sysdump
	if: ${{ always() && steps.install-cilium.outcome != 'skipped' && steps.install-cilium.outcome != 'cancelled' }}
	run: \|
	cilium status
	cilium sysdump \
	--output-filename cilium-sysdump-final \
	--extra-label-selectors=app.kubernetes.io/name=http-perf-client \
	--extra-label-selectors=app.kubernetes.io/name=http-server
	sudo chmod +r cilium-sysdump-final.zip

	- name: Upload sysdump
	if: ${{ !success() && steps.install-cilium.outcome != 'skipped' && steps.install-cilium.outcome != 'cancelled' }}
	uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
	with:
	name: cilium-sysdump-${{ matrix.test_type }}-${{ github.run_attempt }}
	path: cilium-sysdump-final.zip
	retention-days: 5

	- name: Run common post steps
	if: ${{ always() && steps.install-cilium.outcome != 'skipped' }}
	uses: ./.github/actions/post-logic
	with:
	artifacts_suffix: "final-${{ matrix.test_type }}"
	job_status: "${{ job.status }}"
	junits-directory: 'report'
	capture_sysdump: 'false'
	capture_features_tested: 'false'

	- name: Export results and sysdump to GS bucket
	if: ${{ always() && steps.run-cl2.outcome != 'skipped' && steps.run-cl2.outcome != 'cancelled' }}
	uses: cilium/scale-tests-action/export-results@995a08156171644e92e5688b0785b7baf86ce22a # main
	with:
	test_name: ${{ steps.vars.outputs.test_name }}
	tested_sha: ${{ steps.vars.outputs.SHA }}
	results_bucket: ${{ env.GCP_PERF_RESULTS_BUCKET }}
	artifacts: ./perf-tests/clusterloader2/report/
	other_files: cilium-sysdump-final.zip ./perf-tests/clusterloader2/cl2-output.txt

	# Refresh credentials
	- name: Set up AWS CLI credentials
	if: ${{ always() && steps.deploy-cluster.outcome != 'skipped' }}
	uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 # v6.0.0
	with:
	role-to-assume: ${{ secrets.AWS_PR_ASSUME_ROLE }}
	aws-region: ${{ steps.vars.outputs.eks_region }}

	- name: Cleanup cluster
	if: ${{ always() && steps.deploy-cluster.outcome != 'skipped' }}
	run: \|
	eksctl delete cluster --name ${{ steps.vars.outputs.cluster_name }} --region ${{ steps.vars.outputs.eks_region }}

	merge-upload-and-status:
	name: Merge Upload and Status
	if: ${{ always() }}
	needs: install-and-perftest
	uses: ./.github/workflows/common-post-jobs.yaml
	secrets: inherit
	with:
	context-ref: ${{ inputs.context-ref \|\| github.sha }}
	sha: ${{ inputs.SHA \|\| github.sha }}
	success: ${{ needs.install-and-perftest.result == 'success' }}

Navigation Menu

Uh oh!

L7 Policy Perf Test (l7-perf) #104

Workflow file

L7 Policy Perf Test (l7-perf) #104

Uh oh!

Workflow file for this run