Skip to content

Commit 901615f

Browse files
tomaz-s1tomaz-s1
committed
Merge branch 'end_to_end_tests'
2 parents 6ee60da + 3332cd6 commit 901615f

24 files changed

+585
-48
lines changed

‎.github/actions/install-helm-chart/action.yml‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
name: "Install scalyr-agent helm chart action"
2+
description: "Action which installs scalyr-agent helm chart, waits for the pod to fully start up and waits some time for initial logs to be ingested and sent to Scalyr"
3+
4+
inputs:
5+
scalyr_api_key:
6+
description: "Scalyr write API key to use"
7+
required: true
8+
values_file_path:
9+
description: "Path to the values file to use"
10+
required: true
11+
start_sleep_delay:
12+
description: "How many seconds to sleep / wait after installing the chart for the pod to start up"
13+
required: true
14+
default: 20
15+
ingest_sleep_delay:
16+
description: "How many seconds to sleep / wait after the pod has been started up for initial ingestion"
17+
required: true
18+
default: 20
19+
20+
runs:
21+
using: "composite"
22+
steps:
23+
- name: Install Helm chart
24+
shell: bash
25+
run: |
26+
# Set write API key
27+
sed -i "s#REPLACE_ME#${{ inputs.scalyr_api_key }}#g" "${{ inputs.values_file_path }}"
28+
29+
# Install chart
30+
helm install --debug --wait --values "${{ inputs.values_file_path }}" scalyr-agent charts/scalyr-agent
31+
32+
# Verify deployment is either there or not there - depending on the controller type
33+
kubectl get deployments
34+
kubectl get deployment scalyr-agent || true
35+
36+
# Give it some time to start up, technically there is --wait, but it
37+
# doesn't seem to always work
38+
echo "Giving pod some time to start up..."
39+
sleep "${{ inputs.start_sleep_delay }}"
40+
41+
export SCALYR_AGENT_POD_NAME=$(kubectl get pod --selector=test=true -o jsonpath="{.items[0].metadata.name}")
42+
echo "SCALYR_AGENT_POD_NAME=${SCALYR_AGENT_POD_NAME}" >> ${GITHUB_ENV}
43+
44+
echo "Pod logs"
45+
kubectl logs "${SCALYR_AGENT_POD_NAME}"
46+
47+
# Give it some more time for logs to be ingested
48+
echo "Waiting for logs to be ingested..."
49+
sleep "${{ inputs.ingest_sleep_delay }}"

‎.github/actions/setup-chart-testing-environment/action.yml‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
name: "Setup chart testing environment"
2+
description: "Action which sets up Helm chart testing environment, optionally also installing specific Kubernetes version using minikube"
3+
4+
inputs:
5+
minikube_version:
6+
description: "Minikube version to use"
7+
required: false
8+
default: "v1.23.2"
9+
k8s_version:
10+
description: "Kubernetes version to be installed by minikube (if any)"
11+
required: false
12+
default: ""
13+
github_token:
14+
description: "Github token to use for communication with Github API to avoid rate limits"
15+
required: true
16+
17+
runs:
18+
using: "composite"
19+
steps:
20+
- name: Set up Helm
21+
id: setup-helm
22+
uses: azure/setup-helm@18bc76811624f360dbd7f18c2d4ecb32c7b87bab # v1.1
23+
with:
24+
version: v3.7.0
25+
26+
- uses: actions/setup-python@v2
27+
id: setup-python
28+
with:
29+
python-version: 3.7
30+
31+
- name: Set up chart-testing
32+
id: setup-chart-testing
33+
uses: helm/chart-testing-action@5f16c27cf7a4fa9c776ff73734df3909b2b65127 # v2.1.0
34+
with:
35+
version: v3.4.0
36+
37+
- name: Create minikube Kubernetes ${{ inputs.k8s_version }} Cluster
38+
id: create-minikube-cluster
39+
# TODO: Workaround until conditions are supported natively
40+
# See https://github.com/actions/runner/issues/834
41+
uses: ChristopherHX/conditional@b4a9649204f81002ec9a4ef7d4bf7d6b2ab7fa55
42+
with:
43+
if: ${{ inputs.k8s_version != '' }}
44+
step: |
45+
uses: manusa/actions-setup-minikube@3cce81c968cabc530141d5620b7d9942a2907df5 # v2.4.2
46+
with:
47+
minikube version: '${{ inputs.minikube_version }}'
48+
kubernetes version: '${{ inputs.k8s_version }}'
49+
github token: '${{ inputs.github_token }}'
50+
51+
- name: Print minikube environment info
52+
id: print-k8s-cluster-info
53+
uses: ChristopherHX/conditional@b4a9649204f81002ec9a4ef7d4bf7d6b2ab7fa55
54+
with:
55+
if: ${{ inputs.k8s_version != '' }}
56+
step: |
57+
shell: bash
58+
run: |
59+
kubectl -n kube-system rollout restart deployment coredns
60+
kubectl version
61+
minikube addons list
62+
sleep 5

‎.github/workflows/end_to_end_tests.yaml‎

Lines changed: 191 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,191 @@
1+
# Workflow which runs end to end tests against scalyr agent helm chart.
2+
# Those tests verify that the chart can be installed and that it correctly configured agent
3+
# ingestion for various options (daemonset mode, deployment mode, k8s cluster metrics are ingested,
4+
# pod logs are ingested, etc.)
5+
name: "End to End Tests"
6+
7+
on:
8+
push:
9+
branches:
10+
- main
11+
- end_to_end_tests
12+
pull_request:
13+
branches:
14+
- main
15+
- end_to_end_tests
16+
schedule:
17+
- cron: '0 4 * * *'
18+
19+
jobs:
20+
# Special job which automatically cancels old runs for the same branch, prevents runs for the
21+
# same file set which has already passed, etc.
22+
pre_job:
23+
name: Skip Duplicate Jobs Pre Job
24+
runs-on: ubuntu-latest
25+
outputs:
26+
should_skip: ${{ steps.skip_check.outputs.should_skip }}
27+
steps:
28+
- id: skip_check
29+
uses: fkirc/skip-duplicate-actions@f75dd6564bb646f95277dc8c3b80612e46a4a1ea # v3.4.1
30+
with:
31+
cancel_others: 'true'
32+
github_token: ${{ github.token }}
33+
34+
daemonset_controller_type:
35+
name: Daemonset - k8s ${{ matrix.k8s_version }}
36+
runs-on: ubuntu-latest
37+
38+
needs: pre_job
39+
# NOTE: We always want to run job on main branch
40+
if: ${{ needs.pre_job.outputs.should_skip != 'true' || github.ref == 'refs/heads/main' }}
41+
42+
strategy:
43+
fail-fast: false
44+
matrix:
45+
k8s_version:
46+
- 'v1.16.4'
47+
- 'v1.17.2'
48+
- 'v1.22.2'
49+
50+
steps:
51+
- name: Checkout Repository
52+
uses: actions/checkout@v2
53+
with:
54+
fetch-depth: 0
55+
56+
- name: Set up Chart Testing Environment and Kubernetes Cluster
57+
uses: ./.github/actions/setup-chart-testing-environment/
58+
with:
59+
k8s_version: "${{ matrix.k8s_version }}"
60+
github_token: "${{ secrets.GITHUB_TOKEN }}"
61+
62+
- name: Install Scalyr tool
63+
run: |
64+
curl https://raw.githubusercontent.com/scalyr/scalyr-tool/master/scalyr > scalyr
65+
chmod +x scalyr
66+
sudo mv scalyr /usr/local/bin
67+
68+
- name: Install Helm Chart
69+
uses: ./.github/actions/install-helm-chart
70+
with:
71+
scalyr_api_key: "${{ secrets.SCALYR_WRITE_API_KEY_US }}"
72+
values_file_path: "ci/daemonset-agent-values.yaml"
73+
74+
- name: Verify Agent Logs are Ingested
75+
env:
76+
scalyr_readlog_token: "${{ secrets.SCALYR_READ_API_KEY_US }}"
77+
SCALYR_AGENT_POD_NAME: "${{ env.SCALYR_AGENT_POD_NAME }}"
78+
run: |
79+
# Verify agent and kubernetes monitor has been started
80+
./ci/scripts/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile="/var/log/scalyr-agent-2/agent.log" "Starting scalyr agent..."'
81+
./ci/scripts/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile="/var/log/scalyr-agent-2/agent.log" "No checkpoints were found. All logs will be copied starting at their current end"'
82+
83+
./ci/scripts/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile="/var/log/scalyr-agent-2/agent.log" "Cluster name detected, enabling k8s metric reporting"'
84+
./ci/scripts/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile="/var/log/scalyr-agent-2/agent.log" "kubernetes_monitor parameters: "'
85+
86+
# Verify Kubernetes metrics are beeing ingested
87+
./ci/scripts/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile="/var/log/scalyr-agent-2/kubernetes_monitor.log" "k8s-daemon-set=\"scalyr-agent\""'
88+
89+
# We install Redis helm chart and later verify that logs from Redis pods are successfully ingested
90+
- name: Install Redis Helm Chart
91+
run: |
92+
helm repo add bitnami https://charts.bitnami.com/bitnami
93+
helm install --wait --values ci/redis-values.yaml redis-test bitnami/redis
94+
95+
sleep 20
96+
97+
- name: Verify Redis Pods Logs are Ingested
98+
env:
99+
scalyr_readlog_token: "${{ secrets.SCALYR_READ_API_KEY_US }}"
100+
SCALYR_AGENT_POD_NAME: "${{ env.SCALYR_AGENT_POD_NAME }}"
101+
run: |
102+
# master pod
103+
./ci/scripts/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile contains "redis" pod_name="redis-test-master-0" "Redis is starting"'
104+
./ci/scripts/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile contains "redis" pod_name="redis-test-master-0" "Configuration loaded"'
105+
106+
# replica pod
107+
./ci/scripts/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile contains "redis" pod_name="redis-test-replicas-0" "MASTER <-> REPLICA sync started"'
108+
109+
# TODO: Enable once the repo has been moved
110+
# - name: Notify Slack on Failure
111+
# # NOTE: github.ref is set to pr ref (and not branch name, e.g. refs/pull/28/merge) for pull
112+
# # requests and that's why we need this special conditional and check for github.head_ref in
113+
# # case of PRs
114+
# if: ${{ failure() && (github.ref == 'refs/heads/main' || github.head_ref == 'main') }}
115+
# uses: act10ns/slack@e4e71685b9b239384b0f676a63c32367f59c2522 # v1.2.2
116+
# env:
117+
# SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
118+
# with:
119+
# status: ${{ job.status }}
120+
# steps: ${{ toJson(steps) }}
121+
# channel: '#cloud-tech'
122+
123+
deployment_controller_type:
124+
name: Deployment - k8s ${{ matrix.k8s_version }}
125+
runs-on: ubuntu-latest
126+
127+
needs: pre_job
128+
# NOTE: We always want to run job on main branch
129+
if: ${{ needs.pre_job.outputs.should_skip != 'true' || github.ref == 'refs/heads/main' }}
130+
131+
strategy:
132+
fail-fast: false
133+
matrix:
134+
k8s_version:
135+
- 'v1.16.4'
136+
- 'v1.17.2'
137+
- 'v1.22.2'
138+
139+
steps:
140+
- name: Checkout Repository
141+
uses: actions/checkout@v2
142+
with:
143+
fetch-depth: 0
144+
145+
- name: Set up Chart Testing Environment and Kubernetes Cluster
146+
uses: ./.github/actions/setup-chart-testing-environment/
147+
with:
148+
k8s_version: "${{ matrix.k8s_version }}"
149+
github_token: "${{ secrets.GITHUB_TOKEN }}"
150+
151+
- name: Install Scalyr tool
152+
run: |
153+
curl https://raw.githubusercontent.com/scalyr/scalyr-tool/master/scalyr > scalyr
154+
chmod +x scalyr
155+
sudo mv scalyr /usr/local/bin
156+
157+
- name: Install Helm Chart
158+
uses: ./.github/actions/install-helm-chart
159+
with:
160+
scalyr_api_key: "${{ secrets.SCALYR_WRITE_API_KEY_US }}"
161+
values_file_path: "ci/deployment-agent-values.yaml"
162+
163+
- name: Verify Logs are Ingested
164+
env:
165+
scalyr_readlog_token: "${{ secrets.SCALYR_READ_API_KEY_US }}"
166+
SCALYR_AGENT_POD_NAME: "${{ env.SCALYR_AGENT_POD_NAME }}"
167+
run: |
168+
export POD_NAME=$(kubectl get pod --selector=test=true -o jsonpath="{.items[0].metadata.name}")
169+
170+
# Verify agent and kubernetes monitor has been started
171+
./ci/scripts/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile="/var/log/scalyr-agent-2/agent.log" "Starting scalyr agent..."'
172+
./ci/scripts/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile="/var/log/scalyr-agent-2/agent.log" "No checkpoints were found. All logs will be copied starting at their current end"'
173+
174+
./ci/scripts/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile="/var/log/scalyr-agent-2/agent.log" "Cluster name detected, enabling k8s metric reporting"'
175+
./ci/scripts/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile="/var/log/scalyr-agent-2/agent.log" "kubernetes_monitor parameters: "'
176+
177+
# Verify Kubernetes metrics are beeing ingested
178+
./ci/scripts/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile="/var/log/scalyr-agent-2/kubernetes_monitor.log" "k8s-deployment=\"scalyr-agent\""'
179+
180+
- name: Notify Slack on Failure
181+
# NOTE: github.ref is set to pr ref (and not branch name, e.g. refs/pull/28/merge) for pull
182+
# requests and that's why we need this special conditional and check for github.head_ref in
183+
# case of PRs
184+
if: ${{ failure() && (github.ref == 'refs/heads/main' || github.head_ref == 'main') }}
185+
uses: act10ns/slack@e4e71685b9b239384b0f676a63c32367f59c2522 # v1.2.2
186+
env:
187+
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
188+
with:
189+
status: ${{ job.status }}
190+
steps: ${{ toJson(steps) }}
191+
channel: '#cloud-tech'

0 commit comments

Comments
 (0)