Skip to content

Commit 427acec

Browse files
tomaz-s1tomaz-s1
authored
Merge pull request #19 from scalyr/kubernetes_explorer
Add new scalyr.k8s.enableExplorer chart config option which enables Kubernetes Explorer
2 parents f7b8640 + 7e2699a commit 427acec

File tree

12 files changed

+215
-8
lines changed

12 files changed

+215
-8
lines changed

‎.github/actions/setup-chart-testing-environment/action.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ inputs:
55
minikube_version:
66
description: "Minikube version to use"
77
required: false
8-
default: "v1.25.1"
8+
default: "v1.26.1"
99
k8s_version:
1010
description: "Kubernetes version to be installed by minikube (if any)"
1111
required: false
@@ -42,7 +42,7 @@ runs:
4242
with:
4343
if: ${{ inputs.k8s_version != '' }}
4444
step: |
45-
uses: manusa/actions-setup-minikube@3cce81c968cabc530141d5620b7d9942a2907df5 # v2.4.2
45+
uses: manusa/actions-setup-minikube@a36e52547d6fb013f8873758b695ae97a353377b # v2.7.0
4646
with:
4747
minikube version: '${{ inputs.minikube_version }}'
4848
kubernetes version: '${{ inputs.k8s_version }}'

‎.github/workflows/end_to_end_tests.yaml‎

Lines changed: 114 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -46,9 +46,9 @@ jobs:
4646
fail-fast: false
4747
matrix:
4848
k8s_version:
49-
- 'v1.17.2'
5049
- 'v1.20.15'
5150
- 'v1.22.2'
51+
- 'v1.24.3'
5252
image_type:
5353
- "buster"
5454
- "alpine"
@@ -148,9 +148,9 @@ jobs:
148148
fail-fast: false
149149
matrix:
150150
k8s_version:
151-
- 'v1.16.4'
152151
- 'v1.17.2'
153152
- 'v1.22.2'
153+
- 'v1.24.3'
154154
image_type:
155155
- "buster"
156156
- "alpine"
@@ -209,3 +209,115 @@ jobs:
209209
status: ${{ job.status }}
210210
steps: ${{ toJson(steps) }}
211211
channel: '#cloud-tech'
212+
213+
daemonset_controller_type_k8s_explorer:
214+
name: Daemonset (Explorer) - k8s ${{ matrix.k8s_version }} - ${{ matrix.image_type }}
215+
runs-on: ubuntu-latest
216+
217+
needs: pre_job
218+
# NOTE: We always want to run job on main branch
219+
if: ${{ needs.pre_job.outputs.should_skip != 'true' || github.ref == 'refs/heads/main' }}
220+
221+
strategy:
222+
fail-fast: false
223+
matrix:
224+
k8s_version:
225+
- 'v1.22.2'
226+
- 'v1.24.3'
227+
image_type:
228+
- "buster"
229+
- "alpine"
230+
231+
steps:
232+
- name: Checkout Repository
233+
uses: actions/checkout@v2
234+
with:
235+
fetch-depth: 0
236+
237+
- name: Set up Chart Testing Environment and Kubernetes Cluster
238+
uses: ./.github/actions/setup-chart-testing-environment/
239+
with:
240+
k8s_version: "${{ matrix.k8s_version }}"
241+
github_token: "${{ secrets.GITHUB_TOKEN }}"
242+
243+
- name: Install Scalyr tool
244+
run: |
245+
curl https://raw.githubusercontent.com/scalyr/scalyr-tool/master/scalyr > scalyr
246+
chmod +x scalyr
247+
sudo mv scalyr /usr/local/bin
248+
249+
- name: Install Helm Chart
250+
uses: ./.github/actions/install-helm-chart
251+
with:
252+
scalyr_api_key: "${{ secrets.SCALYR_WRITE_API_KEY_US }}"
253+
values_file_path: "ci/daemonset-agent-values-with-k8s-explorer.yaml"
254+
image_type: "${{ matrix.image_type }}"
255+
256+
# Create mock pods and exporters which will be scrapped by the monitor
257+
# (step taken from scalyr-agent-2 repo)
258+
- name: Create mock pods and exporters
259+
run: |
260+
git clone --depth 1 https://github.com/scalyr/scalyr-agent-2.git
261+
262+
kubectl create namespace monitoring
263+
264+
# 1. node exporter pod
265+
pushd scalyr-agent-2/
266+
kubectl apply -f tests/e2e/k8s_om_monitor/node_exporter.yaml
267+
268+
# 2. kube state metrics deployment
269+
kubectl apply -k tests/e2e/k8s_om_monitor/kube-state-metrics/
270+
271+
# 3. Install dummy java app container with jmx exporter side
272+
kubectl apply -f tests/e2e/k8s_om_monitor/java_app_deployment.yaml
273+
popd
274+
275+
sleep 20
276+
kubectl get pods -A
277+
278+
- name: Verify Agent Logs are Ingested
279+
env:
280+
scalyr_readlog_token: "${{ secrets.SCALYR_READ_API_KEY_US }}"
281+
SCALYR_AGENT_POD_NAME: "${{ env.SCALYR_AGENT_POD_NAME }}"
282+
run: |
283+
# Verify agent and kubernetes monitor has been started
284+
./ci/scripts/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile="/var/log/scalyr-agent-2/agent.log" "Starting scalyr agent..."'
285+
./ci/scripts/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile="/var/log/scalyr-agent-2/agent.log" "No checkpoints were found. All logs will be copied starting at their current end"'
286+
287+
./ci/scripts/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile="/var/log/scalyr-agent-2/agent.log" "Cluster name detected, enabling k8s metric reporting"'
288+
./ci/scripts/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile="/var/log/scalyr-agent-2/agent.log" "kubernetes_monitor parameters: "'
289+
290+
#./ci/scripts/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile="/var/log/scalyr-agent-2/agent.log" "Config option '\'stop_agent_on_failure\'' is enabled"'
291+
./ci/scripts/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile="/var/log/scalyr-agent-2/agent.log" "stop_agent_on_failure"'
292+
293+
# Verify Kubernetes metrics are beeing ingested
294+
./ci/scripts/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile="/var/log/scalyr-agent-2/kubernetes_monitor.log" "k8s-daemon-set=\"scalyr-agent\""'
295+
296+
# Verify Kubernetes Explorer logs are there - note: logs won't be complete since atm we
297+
# don't install kube state + node exporter metrics exporter
298+
./ci/scripts/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile="/var/log/scalyr-agent-2/agent.log" "monitor:openmetrics_monitor"'
299+
./ci/scripts/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile="/var/log/scalyr-agent-2/agent.log" "Starting monitor openmetrics_monitor"'
300+
./ci/scripts/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile="/var/log/scalyr-agent-2/agent_debug.log" "Adding new monitor with config:"'
301+
302+
# Assert that Kubernetes Explorer metrics which we scrape from 2 annotated pods are there
303+
./ci/scripts/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" logfile contains "node-exporter" k8s-cluster="k8s-explorer-e2e-tests" k8s-daemon-set="node-exporter" "level=info collector=cpu"'
304+
305+
./ci/scripts/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" monitor="openmetrics_monitor" logfile contains "kubernetes-api-cadvisor-metrics.log" k8s-cluster="k8s-explorer-e2e-tests"'
306+
./ci/scripts/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" monitor="openmetrics_monitor" logfile contains "kubernetes-api-cadvisor-metrics.log" k8s-cluster="k8s-explorer-e2e-tests" "container_start_time_seconds "'
307+
./ci/scripts/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" monitor="openmetrics_monitor" logfile contains "kubernetes-api-cadvisor-metrics.log" k8s-cluster="k8s-explorer-e2e-tests" "container_spec_memory_limit_bytes "'
308+
309+
./ci/scripts/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" monitor="openmetrics_monitor" logfile contains "node-exporter" k8s-cluster="k8s-explorer-e2e-tests"'
310+
./ci/scripts/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" monitor="openmetrics_monitor" logfile contains "node-exporter" k8s-cluster="k8s-explorer-e2e-tests" "process_resident_memory_bytes"'
311+
312+
- name: Notify Slack on Failure
313+
# NOTE: github.ref is set to pr ref (and not branch name, e.g. refs/pull/28/merge) for pull
314+
# requests and that's why we need this special conditional and check for github.head_ref in
315+
# case of PRs
316+
if: ${{ failure() && (github.ref == 'refs/heads/main' || github.head_ref == 'main') }}
317+
uses: act10ns/slack@e4e71685b9b239384b0f676a63c32367f59c2522 # v1.2.2
318+
env:
319+
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
320+
with:
321+
status: ${{ job.status }}
322+
steps: ${{ toJson(steps) }}
323+
channel: '#cloud-tech'

‎CHANGELOG.md‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,23 @@
22

33
For actual scalyr agent changelog, please see https://github.com/scalyr/scalyr-agent-2/blob/release/CHANGELOG.md
44

5+
## 0.2.17
6+
7+
- Add new ``scalyr.k8s.enableExplorer`` chart config option. When this option is set to true, it
8+
enabled Kubernetes Explorer (https://www.dataset.com/blog/introducing-dataset-kubernetes-explorer/)
9+
functionality.
10+
11+
For more information on this functionality, please refer to the docs - https://app.scalyr.com/help/scalyr-agent-k8s-explorer.
12+
13+
- Make sure ``kubernetes_monitor`` has ``stop_agent_on_failure`` monitor config option set to
14+
``true`` (which is the default upstream value).
15+
16+
- Add new ``scalyr.debugLevel`` and ``scalyr.ingestDebugLog`` chart config option which enables
17+
debug logging + debug log ingestion which can help with troubleshooting.
18+
19+
- Update default ``ClusterRole`` definition so it also grants get permissions to ``nodes/proxy``
20+
and ``/metrics`` endpoints which are required for the Kubernetes Explorer functionality.
21+
522
## 0.2.16
623

724
- Update agent to the latest stable version (v2.1.33).

‎README.md‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -112,16 +112,19 @@ For chart changelog, please see <https://github.com/scalyr/helm-scalyr/blob/main
112112
| scalyr.apiKey | string | `""` | The Scalyr API key to use |
113113
| scalyr.base64Config | bool | `true` | As Helm is currently [unable to correctly pass JSON strings](https://github.com/helm/helm/issues/5618), this can be set to true so all values of scalyr.config are expected to be base64 encoded and will be decoded in the chart |
114114
| scalyr.config | object | `{}` | A hash of configuration files and their content as documented in the [Scalyr agent configmap configuration documentation](https://app.scalyr.com/help/scalyr-agent-k8s#modify-config) |
115+
| scalyr.debugLevel | int | `0` | Set this to number between 1 and 5 (inclusive to enable additional debug logging into agent_debug.log). NOTE: If you want this debug log file to be ingested into Scalyr, you should also set scalyr.ingestDebugLog option to true. |
116+
| scalyr.ingestDebugLog | bool | `false` | |
115117
| scalyr.k8s.caCert | string | `""` | The path to the CA certificate to use to verify TLS-connection to the kubelet |
116118
| scalyr.k8s.clusterName | string | `""` | The kubernetes cluster name (when using the kubernetes monitoring) |
117119
| scalyr.k8s.enableEvents | bool | `true` | Enable fetching Kubernetes events |
120+
| scalyr.k8s.enableExplorer | bool | `false` | Enable Kubernetes Explorer functionality (https://www.dataset.com/blog/introducing-dataset-kubernetes-explorer/). This functionality may require additional setup, for more information, please refer to the docs - https://app.scalyr.com/help/scalyr-agent-k8s-explorer NOTE: Explorer functionality is only supported when using DaemonSet agent deployment model. |
118121
| scalyr.k8s.enableLogs | bool | `true` | Enable fetching Pod/Container logs from Kubernetes |
119122
| scalyr.k8s.enableMetrics | bool | `true` | Enable fetching Kubernetes metrics. This requires scalyr.k8s.enableLogs to be true |
120123
| scalyr.k8s.verifyKubeletQueries | bool | `true` | Set this to false to disable TLS cert validation of queries to k8s kubelet. By default cert validation is enabled and connection is verified using the CA configured via the service account certificate (/run/secrets/kubernetes.io/serviceaccount/ca.crt file). If you want to use a custom CA bundle, you can do that by setting scalyr.k8s.caCert config option to point to this file (this file needs to be available inside the agent container). In some test environments such as minikube where self signed certs are used you may want to set this to false. |
121124
| scalyr.server | string | `"agent.scalyr.com"` | The Scalyr server to send logs to. Use eu.scalyr.com for EU |
122125
| securityContext | object | `{}` | optional security context entries |
123-
| tolerations | list | `[{"effect":"NoSchedule","key":"node-role.kubernetes.io/master","operator":"Exists"}]` | Pod tolerations. Defaults to the values documented in the official [Installation guide](https://app.scalyr.com/help/install-agent-kubernetes) |
124126
| serviceAccount.annotations | object | `{}` | optional arbitrary service account annotations |
127+
| tolerations | list | `[{"effect":"NoSchedule","key":"node-role.kubernetes.io/master","operator":"Exists"}]` | Pod tolerations. Defaults to the values documented in the official [Installation guide](https://app.scalyr.com/help/install-agent-kubernetes) |
125128
| volumeMounts | object | `{}` | Additional volume mounts to set up |
126129
| volumes | object | `{}` | Additional volumes to mount |
127130

‎charts/scalyr-agent/Chart.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ apiVersion: v2
22
name: scalyr-agent
33
description: A Helm chart for deploying the Scalyr agent
44
type: application
5-
version: 0.2.16
5+
version: 0.2.17
66
appVersion: 2.1.33
77
keywords:
88
- scalyr

‎charts/scalyr-agent/templates/cluster-role.yaml‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
{{- if or (.Values.scalyr.k8s.enableMetrics) (.Values.scalyr.k8s.enableLogs) }}
1+
{{- if (or (or (.Values.scalyr.k8s.enableMetrics) (.Values.scalyr.k8s.enableLogs)) .Values.scalyr.k8s.enableExplorer) }}
22
apiVersion: rbac.authorization.k8s.io/v1
33
kind: ClusterRole
44
metadata:
@@ -51,10 +51,17 @@ rules:
5151
- "get"
5252
- "list"
5353
- "watch"
54+
# nodes/proxy and /metrics permissions are needed for Kubernetes Open Metrics monitor
55+
# so local Kubelet API can be queried for metrics in Open Metrics format.
5456
- apiGroups:
5557
- ""
5658
resources:
5759
- "nodes/stats"
60+
- "nodes/proxy"
61+
verbs:
62+
- "get"
63+
- nonResourceURLs:
64+
- "/metrics"
5865
verbs:
5966
- "get"
6067
{{- end }}

‎charts/scalyr-agent/templates/configmap.yaml‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,25 @@ data:
1717
{{- if .Values.scalyr.k8s.enableLogs }}
1818
{
1919
"module": "scalyr_agent.builtin_monitors.kubernetes_monitor",
20+
"stop_agent_on_failure": true,
2021
"report_container_metrics": {{ .Values.scalyr.k8s.enableMetrics }},
2122
"report_k8s_metrics": {{ .Values.scalyr.k8s.enableMetrics }}
23+
},
24+
{{- end }}
25+
{{- if .Values.scalyr.k8s.enableExplorer }}
26+
{
27+
module: "scalyr_agent.builtin_monitors.kubernetes_openmetrics_monitor",
28+
include_node_name: true,
29+
include_cluster_name: true,
30+
logger_include_node_name: false,
31+
// Switch to true once preview phase is over
32+
verify_https: false,
33+
scrape_kubernetes_api_metrics: false,
34+
kubernetes_api_metrics_scrape_interval: 60,
35+
scrape_kubernetes_api_cadvisor_metrics: true,
36+
kubernetes_api_cadvisor_metrics_scrape_interval: 60,
37+
sample_interval: 60,
38+
scrape_interval: 60
2239
}
2340
{{- end }}
2441
]
@@ -34,3 +51,17 @@ data:
3451
{{- end }}
3552
{{ if $.Values.scalyr.base64Config }}{{ $value | b64dec }}{{ else }}{{ $value }}{{ end }}
3653
{{- end }}
54+
{{- if .Values.scalyr.ingestDebugLog }}
55+
"debug_log.json": |
56+
{
57+
"logs": [
58+
{
59+
"path": "/var/log/scalyr-agent-2/agent_debug.log",
60+
"attributes":
61+
{
62+
"parser": "scalyrAgentLog"
63+
},
64+
},
65+
]
66+
}
67+
{{- end }}

‎charts/scalyr-agent/templates/daemonset.yaml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,14 @@ spec:
9191
- name: "SCALYR_K8S_KUBELET_CA_CERT"
9292
value: "{{ .Values.scalyr.k8s.caCert }}"
9393
{{- end }}
94+
{{- if (.Values.scalyr.k8s.enableExplorer) }}
95+
- name: "SCALYR_K8S_EXPLORER_ENABLE"
96+
value: "true"
97+
{{- end }}
98+
{{- if (not (eq (int .Values.scalyr.debugLevel) 0)) }}
99+
- name: "SCALYR_DEBUG_LEVEL"
100+
value: "{{ .Values.scalyr.debugLevel }}"
101+
{{- end }}
94102
- name: "SCALYR_K8S_NODE_NAME"
95103
valueFrom:
96104
fieldRef:

‎charts/scalyr-agent/templates/deployment.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,10 @@ spec:
9292
- name: "SCALYR_K8S_KUBELET_CA_CERT"
9393
value: "{{ .Values.scalyr.k8s.caCert }}"
9494
{{- end }}
95+
{{- if (not (eq (int .Values.scalyr.debugLevel) 0)) }}
96+
- name: "SCALYR_DEBUG_LEVEL"
97+
value: "{{ .Values.scalyr.debugLevel }}"
98+
{{- end }}
9599
- name: "SCALYR_K8S_NODE_NAME"
96100
valueFrom:
97101
fieldRef:

‎charts/scalyr-agent/values.yaml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,12 @@ scalyr:
1919
server: "agent.scalyr.com"
2020
# scalyr.apiKey -- The Scalyr API key to use
2121
apiKey: ""
22+
# scalyr.debugLevel -- Set this to number between 1 and 5 (inclusive to enable additional debug
23+
# logging into agent_debug.log).
24+
# NOTE: If you want this debug log file to be ingested into Scalyr, you should also set
25+
# scalyr.ingestDebugLog option to true.
26+
debugLevel: 0
27+
ingestDebugLog: false
2228
k8s:
2329
# scalyr.k8s.clusterName -- The kubernetes cluster name (when using the kubernetes monitoring)
2430
clusterName: ""
@@ -37,6 +43,11 @@ scalyr:
3743
enableMetrics: true
3844
# scalyr.k8s.enableEvents -- Enable fetching Kubernetes events
3945
enableEvents: true
46+
# scalyr.k8s.enableExplorer -- Enable Kubernetes Explorer functionality (https://www.dataset.com/blog/introducing-dataset-kubernetes-explorer/).
47+
# This functionality may require additional setup, for more information, please refer to the
48+
# docs - https://app.scalyr.com/help/scalyr-agent-k8s-explorer
49+
# NOTE: Explorer functionality is only supported when using DaemonSet agent deployment model.
50+
enableExplorer: false
4051
# scalyr.config -- A hash of configuration files and their content as documented in the
4152
# [Scalyr agent configmap configuration documentation](https://app.scalyr.com/help/scalyr-agent-k8s#modify-config)
4253
config: {}

0 commit comments

Comments
 (0)