diff --git a/CHANGELOG.md b/CHANGELOG.md index 3d4a85a99..1bc412446 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ - Implemented automatic data migration from Bitnami deployments (PVC reuse) - Enabled AOF persistence by default for data durability - add CounterBasedGauge64 and ZeroBasedCounter64 as metrics types +- add SNMP-enabled device discovery feature ### Fixes - fix problem with service rendering when `traps.service.usemetallb` is set to false diff --git a/Dockerfile b/Dockerfile index 9ca6c431e..676e88884 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,6 +5,7 @@ ENV PYTHONFAULTHANDLER=1 \ PYTHONUNBUFFERED=1 RUN apk add -U git sqlite-dev RUN pip install --upgrade setuptools pip +RUN apk add --no-cache nmap RUN mkdir /app WORKDIR /app diff --git a/charts/splunk-connect-for-snmp/templates/NOTES.txt b/charts/splunk-connect-for-snmp/templates/NOTES.txt index 94fd81372..ee7ac9d34 100644 --- a/charts/splunk-connect-for-snmp/templates/NOTES.txt +++ b/charts/splunk-connect-for-snmp/templates/NOTES.txt @@ -1,2 +1,3 @@ Default walk no longer calls full oid tree, instead it is collecting only 'SNMPv2-MIB'. -If you want to call full oid for the devices, you have to set enableFullWalk flag to true. \ No newline at end of file +If you want to call full oid for the devices, you have to set enableFullWalk flag to true. +If the discovery feature is enabled, the time required may increase depending on the number of devices in the specified subnet. diff --git a/charts/splunk-connect-for-snmp/templates/_helpers.tpl b/charts/splunk-connect-for-snmp/templates/_helpers.tpl index fcf942715..e5d6768d2 100644 --- a/charts/splunk-connect-for-snmp/templates/_helpers.tpl +++ b/charts/splunk-connect-for-snmp/templates/_helpers.tpl @@ -94,4 +94,15 @@ Whether enable polling {{- else }} {{- printf "false" }} {{- end -}} +{{- end }} + +{{/* +Whether enable discovery +*/}} +{{- define "splunk-connect-for-snmp.discovery.enable" -}} +{{- if .Values.discovery.enabled }} +{{- printf "true" }} +{{- else }} +{{- printf "false" }} +{{- end -}} {{- end }} \ No newline at end of file diff --git a/charts/splunk-connect-for-snmp/templates/common/discovery-config.yaml b/charts/splunk-connect-for-snmp/templates/common/discovery-config.yaml new file mode 100644 index 000000000..39460f953 --- /dev/null +++ b/charts/splunk-connect-for-snmp/templates/common/discovery-config.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "splunk-connect-for-snmp.name" . }}-discovery-config +data: + discovery-config.yaml: | +{{- if .Values.discovery }} +{{ .Values.discovery | toYaml | indent 4 }} +{{- end }} diff --git a/charts/splunk-connect-for-snmp/templates/discovery/_helpers.tpl b/charts/splunk-connect-for-snmp/templates/discovery/_helpers.tpl new file mode 100644 index 000000000..91f35da53 --- /dev/null +++ b/charts/splunk-connect-for-snmp/templates/discovery/_helpers.tpl @@ -0,0 +1,50 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "splunk-connect-for-snmp.discovery.name" -}} +{{- default (printf "%s-%s" .Chart.Name "discovery") .Values.discovery.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "splunk-connect-for-snmp.discovery.fullname" -}} +{{- if .Values.discovery.fullnameOverride }} +{{- .Values.discovery.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default (printf "%s-%s" .Chart.Name "discovery") .Values.discovery.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Not using it anywhere +*/}} +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "splunk-connect-for-snmp.discovery.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "splunk-connect-for-snmp.discovery.selectorLabels" -}} +app.kubernetes.io/name: {{ include "splunk-connect-for-snmp.discovery.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "splunk-connect-for-snmp.discovery.labels" -}} +{{ include "splunk-connect-for-snmp.discovery.selectorLabels" . }} +{{ include "splunk-connect-for-snmp.labels" . }} +{{- end }} diff --git a/charts/splunk-connect-for-snmp/templates/discovery/job.yaml b/charts/splunk-connect-for-snmp/templates/discovery/job.yaml new file mode 100644 index 000000000..a92d01832 --- /dev/null +++ b/charts/splunk-connect-for-snmp/templates/discovery/job.yaml @@ -0,0 +1,78 @@ +{{- if eq (include "splunk-connect-for-snmp.discovery.enable" .) "true" }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "splunk-connect-for-snmp.discovery.fullname" . }} + labels: + {{- include "splunk-connect-for-snmp.discovery.labels" . | nindent 4 }} +spec: + ttlSecondsAfterFinished: 300 + template: + metadata: + # {{- with .Values.inventory.podAnnotations }} + # annotations: + # {{- toYaml . | nindent 8 }} + # {{- end }} + + labels: + {{- include "splunk-connect-for-snmp.discovery.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: {{ .Chart.Name }}-discovery + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + ["discovery"] + env: + {{- if .Values.redis.auth.enabled }} + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + {{- if .Values.redis.auth.existingSecret }} + name: {{ .Values.redis.auth.existingSecret }} + key: {{ .Values.redis.auth.existingSecretPasswordKey | default "password" }} + {{- else }} + name: {{ .Release.Name }}-redis-secret + key: password + {{- end }} + {{- end }} + - name: REDIS_HOST + value: {{ .Release.Name }}-redis + - name: REDIS_PORT + value: "6379" + - name: REDIS_DB + value: "1" + - name: CELERY_DB + value: "0" + - name: DISCOVERY_CONFIG_PATH + value: /app/discovery/discovery-config.yaml + - name: LOG_LEVEL + value: {{ .Values.discovery.logLevel | default "INFO" }} + - name: CHAIN_OF_TASKS_EXPIRY_TIME + value: {{ .Values.scheduler.tasksExpiryTime | quote }} + - name: CELERY_TASK_TIMEOUT + value: {{ .Values.worker.taskTimeout | quote}} + volumeMounts: + - name: discovery-config + mountPath: "/app/discovery" + readOnly: true + - name: tmp + mountPath: "/tmp/" + readOnly: false + + volumes: + # # You set volumes at the Pod level, then mount them into containers inside that Pod + - name: discovery-config + configMap: + name: {{ include "splunk-connect-for-snmp.name" . }}-discovery-config + items: + - key: "discovery-config.yaml" + path: "discovery-config.yaml" + - name: tmp + emptyDir: {} + restartPolicy: OnFailure +{{- end -}} \ No newline at end of file diff --git a/charts/splunk-connect-for-snmp/templates/worker/_helpers.tpl b/charts/splunk-connect-for-snmp/templates/worker/_helpers.tpl index 145986cb1..eb7c45dbe 100644 --- a/charts/splunk-connect-for-snmp/templates/worker/_helpers.tpl +++ b/charts/splunk-connect-for-snmp/templates/worker/_helpers.tpl @@ -59,6 +59,10 @@ app.kubernetes.io/name: {{ include "splunk-connect-for-snmp.worker.name" . }}-fl app.kubernetes.io/instance: {{ .Release.Name }} {{- end }} +{{- define "splunk-connect-for-snmp.worker.discovery.selectorLabels" -}} +app.kubernetes.io/name: {{ include "splunk-connect-for-snmp.worker.name" . }}-discovery +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} {{/* Common labels @@ -88,6 +92,11 @@ Common labels {{ include "splunk-connect-for-snmp.labels" . }} {{- end }} +{{- define "splunk-connect-for-snmp.worker.discovery.labels" -}} +{{ include "splunk-connect-for-snmp.worker.discovery.selectorLabels" . }} +{{ include "splunk-connect-for-snmp.labels" . }} +{{- end }} + {{- define "environmental-variables" -}} - name: CONFIG_PATH value: /app/config/config.yaml @@ -125,6 +134,8 @@ Common labels value: {{ .Values.worker.disableMongoDebugLogging | quote }} - name: UDP_CONNECTION_TIMEOUT value: {{ .Values.worker.udpConnectionTimeout | default "3" | quote }} +- name: UDP_CONNECTION_RETRIES + value: {{ .Values.worker.udpConnectionRetries | default "5" | quote }} - name: MAX_OID_TO_PROCESS value: {{ .Values.poller.maxOidToProcess | default "70" | quote }} - name: MAX_REPETITIONS @@ -217,4 +228,11 @@ Common labels {{ else }} value: "false" {{- end }} +{{- end }} + +{{- define "environmental-variables-discovery" -}} +- name: WORKER_CONCURRENCY + value: {{ .Values.worker.discovery.concurrency | default "4" | quote }} +- name: PREFETCH_COUNT + value: {{ .Values.worker.discovery.prefetch | default "30" | quote }} {{- end }} \ No newline at end of file diff --git a/charts/splunk-connect-for-snmp/templates/worker/discovery/deployment.yaml b/charts/splunk-connect-for-snmp/templates/worker/discovery/deployment.yaml new file mode 100644 index 000000000..9d6f9314d --- /dev/null +++ b/charts/splunk-connect-for-snmp/templates/worker/discovery/deployment.yaml @@ -0,0 +1,95 @@ +{{- if eq (include "splunk-connect-for-snmp.discovery.enable" .) "true" }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "splunk-connect-for-snmp.worker.fullname" . }}-discovery + labels: + {{- include "splunk-connect-for-snmp.worker.discovery.labels" . | nindent 4 }} +spec: + {{- if not .Values.worker.discovery.autoscaling.enabled }} + replicas: {{ .Values.worker.discovery.replicaCount }} + {{- end }} + selector: + matchLabels: + {{- include "splunk-connect-for-snmp.worker.discovery.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "splunk-connect-for-snmp.worker.discovery.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "splunk-connect-for-snmp.serviceAccountName" . }} + securityContext: + fsGroup: 10001 + containers: + - name: {{ .Chart.Name }}-discovery + securityContext: + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 10001 + runAsGroup: 10001 + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + [ + "celery", "worker-discovery", + ] + env: + {{- include "environmental-variables" . | nindent 12 }} + - name: DISCOVERY_FOLDER_PATH + value: /app/discovery + - name: CELERY_TASK_TIMEOUT + value: {{ .Values.worker.taskTimeout | quote}} + - name: IPv6_ENABLED + {{- if .Values.discovery.ipv6Enabled}} + value: "true" + {{ else }} + value: "false" + {{- end }} + volumeMounts: + - name: config + mountPath: "/app/config" + readOnly: true + - name: pysnmp-cache-volume + mountPath: "/.pysnmp/" + readOnly: false + - name: tmp + mountPath: "/tmp/" + readOnly: false + - name: discovery-volume + mountPath: /app/discovery + {{- range (.Values.discovery).usernameSecrets }} + - name: {{ . }}-snmpv3-secrets + mountPath: /app/secrets/snmpv3/{{ . }} + readOnly: true + {{- end }} + volumes: + # You set volumes at the Pod level, then mount them into containers inside that Pod + - name: config + configMap: + # Provide the name of the ConfigMap you want to mount. + name: {{ include "splunk-connect-for-snmp.name" . }}-config + # An array of keys from the ConfigMap to create as files + items: + - key: "config.yaml" + path: "config.yaml" + - name: pysnmp-cache-volume + emptyDir: {} + - name: tmp + emptyDir: {} + - name: discovery-volume + hostPath: + path: {{ .Values.discovery.discoveryPath }} + type: Directory + {{- range (.Values.discovery).usernameSecrets }} + - name: {{ . }}-snmpv3-secrets + secret: + secretName: {{ . }} + {{- end }} +{{- end }} diff --git a/charts/splunk-connect-for-snmp/templates/worker/discovery/hpa.yaml b/charts/splunk-connect-for-snmp/templates/worker/discovery/hpa.yaml new file mode 100644 index 000000000..d6d0a0a21 --- /dev/null +++ b/charts/splunk-connect-for-snmp/templates/worker/discovery/hpa.yaml @@ -0,0 +1,24 @@ +{{- if and ( eq (include "splunk-connect-for-snmp.discovery.enable" .) "true" ) (eq ( toString .Values.worker.discovery.autoscaling.enabled) "true") }} +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "splunk-connect-for-snmp.worker.fullname" . }}-discovery + labels: + {{- include "splunk-connect-for-snmp.worker.discovery.labels" . | nindent 4 }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "splunk-connect-for-snmp.worker.fullname" . }}-discovery + minReplicas: {{ .Values.worker.discovery.autoscaling.minReplicas }} + maxReplicas: {{ .Values.worker.discovery.autoscaling.maxReplicas }} + metrics: + {{- if .Values.worker.discovery.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: {{ .Values.worker.discovery.autoscaling.targetCPUUtilizationPercentage | default 80 }} + {{- end }} +{{- end }} diff --git a/charts/splunk-connect-for-snmp/values.schema.json b/charts/splunk-connect-for-snmp/values.schema.json index 3f6a2a3f2..8930b8520 100644 --- a/charts/splunk-connect-for-snmp/values.schema.json +++ b/charts/splunk-connect-for-snmp/values.schema.json @@ -10,7 +10,8 @@ "poller", "worker", "inventory", - "traps" + "traps", + "discovery" ], "title": "Values", "additionalProperties": false, @@ -593,6 +594,69 @@ } } }, + "discovery": { + "type": "object", + "additionalProperties": false, + "properties": { + "replicaCount": { + "type": "integer" + }, + "concurrency": { + "type": "integer" + }, + "prefetch": { + "type": "integer" + }, + "autoscaling": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean" + }, + "minReplicas": { + "type": "integer" + }, + "maxReplicas": { + "type": "integer" + }, + "targetCPUUtilizationPercentage": { + "type": "integer" + } + } + }, + "resources": { + "type": "object", + "additionalProperties": false, + "properties": { + "limits": { + "type": "object", + "additionalProperties": false, + "properties": { + "cpu": { + "type": ["integer", "string"] + }, + "memory": { + "type": ["integer", "string"] + } + } + }, + "requests": { + "type": "object", + "additionalProperties": false, + "properties": { + "cpu": { + "type": ["integer", "string"] + }, + "memory": { + "type": ["integer", "string"] + } + } + } + } + } + } + }, "livenessProbe": { "type": "object", "additionalProperties": false, @@ -663,6 +727,9 @@ "udpConnectionTimeout": { "type": "integer" }, + "udpConnectionRetries": { + "type": "integer" + }, "ignoreEmptyVarbinds": { "type": "boolean" }, @@ -901,6 +968,88 @@ } } } + }, + "discovery": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean" + }, + "ipv6Enabled": { + "type": "boolean" + }, + "usernameSecrets": { + "type": "array" + }, + "discoveryPath": { + "type": "string" + }, + "logLevel": { + "type": "string" + }, + "autodiscovery": { + "type": "object", + "additionalProperties": false, + "patternProperties": { + "^[^0-9].*": { + "type": "object", + "additionalProperties": false, + "properties": { + "frequency": { + "type": "integer" + }, + "delete_already_discovered": { + "type": "boolean" + }, + "network_address": { + "type": "string" + }, + "version": { + "type": "string", + "enum": ["1", "2c", "3"] + }, + "secret": { + "type": ["string", "null"] + }, + "community": { + "type": ["string", "null"] + }, + "port": { + "type": "integer", + "minimum": 1, + "maximum": 65535 + }, + "security_engine": { + "type": "string" + }, + "device_rules": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "patterns": { + "type": "string" + }, + "group": { + "type": "string" + } + }, + "required": [ + "name", + "patterns", + "group" + ] + } + } + } + } + } + } + } } } } \ No newline at end of file diff --git a/charts/splunk-connect-for-snmp/values.yaml b/charts/splunk-connect-for-snmp/values.yaml index 63ca0e067..bfe0213d3 100644 --- a/charts/splunk-connect-for-snmp/values.yaml +++ b/charts/splunk-connect-for-snmp/values.yaml @@ -351,6 +351,32 @@ worker: # the resources requests for sender worker container requests: cpu: 250m + + # The discovery worker handles auto discovery of snmp enabled devices and create CSV file for it + discovery: + # number of the discovery replicas when autoscaling is set to false + replicaCount: 1 + # minimum number of threads in a pod + concurrency: 4 + # how many tasks are consumed from the queue at once + prefetch: 30 + autoscaling: + # enabling autoscaling for discovery worker pods + enabled: false + # minimum number of running discovery worker pods when autoscaling is enabled + minReplicas: 2 + # maximum number of running discovery worker pods when autoscaling is enabled + maxReplicas: 10 + # CPU % threshold that must be exceeded on discovery worker pods to spawn another replica + targetCPUUtilizationPercentage: 80 + resources: + # the resources limits for discovery worker container + limits: + cpu: 500m + # the resources requests for discovery worker container + requests: + cpu: 250m + # Liveness probes are used in Kubernetes to know when a pod is alive or dead. # A pod can be in a dead state for a number of reasons; # the application could be crashed, some error in the application etc. @@ -403,6 +429,8 @@ worker: podAntiAffinity: soft # udpConnectionTimeout timeout in seconds for SNMP operations udpConnectionTimeout: 3 + # udpConnectionRetries number of retries for SNMP operations + udpConnectionRetries: 5 # in case of seeing "Empty SNMP response message" this variable can be set to true ignoreEmptyVarbinds: false @@ -639,3 +667,13 @@ redis: fsGroup: 999 commonAnnotations: {} +discovery: + # Enables discovering SNMP enabled device and create CSV file. + enabled: false + # list of kubernetes secrets name that will be used for discovery + # https://splunk.github.io/splunk-connect-for-snmp/main/microk8s/configuration/discovery-configuration/#define-usernamesecrets + usernameSecrets: [] + # Enabled device detection using IPv6 subnet. + ipv6Enabled: false + # logging level, possible options: DEBUG, INFO, WARNING, ERROR, CRITICAL, or FATAL + logLevel: "INFO" diff --git a/dashboard/dashboard.xml b/dashboard/dashboard.xml index c4588552a..33f5b7e35 100644 --- a/dashboard/dashboard.xml +++ b/dashboard/dashboard.xml @@ -185,6 +185,81 @@ + + + SNMP discovery status + + + all + * + * + key + key + + index=* sourcetype="*:container:splunk-connect-for-snmp-*" "Scheduler: Sending due task sc4snmp;*;discovery" | rex field=_raw "Sending due task sc4snmp;(?<key>.+);discovery" | stats count by key + -24h@h + now + + + + In case of unsuccessful discovery status, please copy SPL query from this chart and find failed tasks. Explanation of error log messages you can find at + https://splunk.github.io/splunk-connect-for-snmp/main/troubleshooting/discovery-issues/ + + index=* sourcetype="*:container:splunk-connect-for-snmp-*" splunk_connect_for_snmp.discovery.tasks.discovery "$discovery_status_key$" | rex field=_raw "Task splunk_connect_for_snmp.*\[*\] (?<status>\w+)" | where status != "received" | timechart count by status + -24h@h + now + 5m + delay + + + + + + + + + + + + + search?q=index%3D*%20sourcetype%3D%22*%3Acontainer%3Asplunk-connect-for-snmp-*%22%20splunk_connect_for_snmp.discovery.tasks.discovery%20%22$discovery_status_key$%22%20%7C%20rex%20field%3D_raw%20%22Task%20splunk_connect_for_snmp.*%5C%5B*%5C%5D%20(%3F%3Cstatus%3E%5Cw%2B)%22%20%7C%20where%20status%20!%3D%20%22received%22&earliest=-24h@h&latest=now + + + + + SNMP schedule of discovery tasks + + + all + * + * + key + key + + index=* sourcetype="*:container:splunk-connect-for-snmp-*" "Scheduler: Sending due task sc4snmp;*;discovery" | rex field=_raw "Sending due task sc4snmp;(?<key>.+);discovery" | stats count by key + -24h@h + now + + + + Using this chart you can understand when SC4SNMP scheduled discovery for your Discovery Key last time. The process works if it runs regularly. + + index=* sourcetype="*:container:splunk-connect-for-snmp-*" Scheduler: Sending due task sc4snmp;*;discovery "$discovery_key$" | timechart count + -24h@h + now + 5m + delay + + + + + + + search?q=index%3D*%20sourcetype%3D%22*%3Acontainer%3Asplunk-connect-for-snmp-*%22%20Scheduler%3A%20Sending%20due%20task%20sc4snmp%3B*%3Bdiscovery%20%22$discovery_key$%22&earliest=-24h@h&latest=now + + + + SNMP send to Splunk status diff --git a/docker_compose/.env b/docker_compose/.env index c48616e4e..4d67ed3b4 100644 --- a/docker_compose/.env +++ b/docker_compose/.env @@ -4,6 +4,7 @@ SC4SNMP_TAG="1.14.2-beta.4" SCHEDULER_CONFIG_FILE_ABSOLUTE_PATH= TRAPS_CONFIG_FILE_ABSOLUTE_PATH= INVENTORY_FILE_ABSOLUTE_PATH= +DISCOVERY_CONFIG_FILE_ABSOLUTE_PATH= COREFILE_ABS_PATH= SC4SNMP_VERSION="1.14.2-beta.4" @@ -16,6 +17,11 @@ IPAM_GATEWAY=172.28.0.1 IPAM_SUBNET_IPv6=fd02::/64 IPAM_GATEWAY_IPv6=fd02::1 +# Discovery +DISCOVERY_ENABLE=true +DISCOVERY_LOG_LEVEL=INFO +DISCOVERY_PATH= + # Dependencies images COREDNS_IMAGE=coredns/coredns COREDNS_TAG=1.11.1 @@ -52,7 +58,9 @@ IGNORE_NOT_INCREASING_OIDS= WORKER_LOG_LEVEL=INFO WORKER_DISABLE_MONGO_DEBUG_LOGGING=true UDP_CONNECTION_TIMEOUT=3 +UDP_CONNECTION_RETRIES=5 MAX_OID_TO_PROCESS=70 +CELERY_TASK_TIMEOUT=2400 MAX_REPETITIONS=10 # Worker Poller @@ -85,6 +93,15 @@ WORKER_TRAP_MEMORY_LIMIT=500M WORKER_TRAP_CPU_RESERVATIONS=0.5 WORKER_TRAP_MEMORY_RESERVATIONS=250M +# Worker Discovery +WORKER_DISCOVERY_CONCURRENCY=4 +PREFETCH_DISCOVERY_COUNT=30 +WORKER_DISCOVERY_REPLICAS=1 +WORKER_DISCOVERY_CPU_LIMIT=1 +WORKER_DISCOVERY_MEMORY_LIMIT=500M +WORKER_DISCOVERY_CPU_RESERVATIONS=0.5 +WORKER_DISCOVERY_MEMORY_RESERVATIONS=250M + # Inventory configuration INVENTORY_LOG_LEVEL=INFO CHAIN_OF_TASKS_EXPIRY_TIME=500 diff --git a/docker_compose/docker-compose.yaml b/docker_compose/docker-compose.yaml index 8a180acff..e4542fde7 100644 --- a/docker_compose/docker-compose.yaml +++ b/docker_compose/docker-compose.yaml @@ -35,6 +35,7 @@ x-workers_general_setup: &workers_general_setup LOG_LEVEL: ${WORKER_LOG_LEVEL:-INFO} DISABLE_MONGO_DEBUG_LOGGING: ${WORKER_DISABLE_MONGO_DEBUG_LOGGING:-true} UDP_CONNECTION_TIMEOUT: ${UDP_CONNECTION_TIMEOUT:-3} + UDP_CONNECTION_RETRIES: ${UDP_CONNECTION_RETRIES:-5} MAX_OID_TO_PROCESS: ${MAX_OID_TO_PROCESS:-70} MAX_REPETITIONS: ${MAX_REPETITIONS:-10} PROFILES_RELOAD_DELAY: ${PROFILES_RELOAD_DELAY:-60} @@ -257,6 +258,52 @@ services: - ${SCHEDULER_CONFIG_FILE_ABSOLUTE_PATH}:/app/config/config.yaml:ro - worker-flower-pysnmp-cache-volume:/.pysnmp/:rw - worker-flower-tmp:/tmp/:rw + discovery: + <<: [ *dns_and_networks, *dependency_and_restart_policy ] + image: ${SC4SNMP_IMAGE}:${SC4SNMP_TAG:-latest} + container_name: sc4snmp-discovery + command: [ discovery ] + environment: + <<: *general_sc4snmp_data + LOG_LEVEL: ${DISCOVERY_LOG_LEVEL:-INFO} + CHAIN_OF_TASKS_EXPIRY_TIME: ${CHAIN_OF_TASKS_EXPIRY_TIME:-500} + CELERY_TASK_TIMEOUT: ${CELERY_TASK_TIMEOUT:-2400} + DISCOVERY_CONFIG_PATH: /app/discovery/discovery-config.yaml + volumes: + - ${SCHEDULER_CONFIG_FILE_ABSOLUTE_PATH}:/app/config/config.yaml:ro + - ${DISCOVERY_CONFIG_FILE_ABSOLUTE_PATH}:/app/discovery/discovery-config.yaml:ro + - discovery-pysnmp-cache-volume:/.pysnmp/:rw + - discovery-tmp:/tmp/:rw + worker-discovery: + <<: [ *dns_and_networks, *dependency_and_restart_policy ] + image: ${SC4SNMP_IMAGE}:${SC4SNMP_TAG:-latest} + command: + - celery + - worker-discovery + deploy: + mode: replicated + replicas: ${WORKER_DISCOVERY_REPLICAS:-1} + resources: + limits: + cpus: ${WORKER_DISCOVERY_CPU_LIMIT:-0.50} + memory: ${WORKER_DISCOVERY_MEMORY_LIMIT:-500M} + reservations: + cpus: ${WORKER_DISCOVERY_CPU_RESERVATIONS:-0.25} + memory: ${WORKER_DISCOVERY_MEMORY_RESERVATIONS:-250M} + environment: + <<: [ *general_sc4snmp_data, *pysnmp_debug, *ipv6 ] + SC4SNMP_VERSION: ${SC4SNMP_VERSION:-latest} + LOG_LEVEL: ${WORKER_LOG_LEVEL:-INFO} + UDP_CONNECTION_TIMEOUT: ${UDP_CONNECTION_TIMEOUT:-3} + UDP_CONNECTION_RETRIES: ${UDP_CONNECTION_RETRIES:-5} + WORKER_CONCURRENCY: ${WORKER_DISCOVERY_CONCURRENCY:-1} + PREFETCH_COUNT: ${PREFETCH_SENDER_COUNT:-1} + DISCOVERY_FOLDER_PATH: /app/discovery + volumes: + - ${SCHEDULER_CONFIG_FILE_ABSOLUTE_PATH}:/app/config/config.yaml:ro + - ${DISCOVERY_PATH}:/app/discovery/:rw + - worker-discovery-pysnmp-cache-volume:/.pysnmp/:rw + - worker-discovery-tmp:/tmp/:rw volumes: snmp-mibserver-tmp: inventory-tmp: @@ -273,3 +320,8 @@ volumes: worker-trap-pysnmp-cache-volume: worker-flower-tmp: worker-flower-pysnmp-cache-volume: + discovery-pysnmp-cache-volume: + discovery-tmp: + worker-discovery-pysnmp-cache-volume: + worker-discovery-tmp: + worker-discovery-volume: diff --git a/docker_compose/manage_secrets.py b/docker_compose/manage_secrets.py index 079d8890f..a88d83c66 100644 --- a/docker_compose/manage_secrets.py +++ b/docker_compose/manage_secrets.py @@ -4,7 +4,7 @@ import ruamel.yaml -SERVICE_SECRETS = ["worker-poller", "traps"] +SERVICE_SECRETS = ["worker-poller", "traps", "worker-discovery"] DOCKER_COMPOSE = "docker-compose.yaml" @@ -60,6 +60,7 @@ def create_secrets( secret_name: str, make_change_in_worker_poller: bool, make_change_in_traps: bool, + make_change_in_worker_discovery: bool, ): """ Function to create secrets in .env and docker-compose.yaml files @@ -68,6 +69,7 @@ def create_secrets( @param secret_name: name of the secret @param make_change_in_worker_poller: flag indicating whether to add secrets to worker poller service @param make_change_in_traps: flag indicating whether to add secrets to traps service + @param make_change_in_worker_discovery: flag indicating whether to add secrets to discovery service """ for k, v in variables.items(): if k != "contextEngineId" and not v: @@ -107,6 +109,13 @@ def create_secrets( else: traps_ready = True + if make_change_in_worker_discovery: + yaml_file, worker_discovery_ready = load_compose_worker_discovery( + new_secrets_in_workers, yaml_file + ) + else: + worker_discovery_ready = True + save_to_compose_files( path_to_compose_files, secret_name, @@ -115,6 +124,7 @@ def create_secrets( traps_ready, variables, worker_poller_ready, + worker_discovery_ready, ) except Exception as e: print(f"Problem with adding secrets. Error: {e}") @@ -128,8 +138,9 @@ def save_to_compose_files( traps_ready, variables, worker_poller_ready, + worker_discovery_ready, ): - if secrets_ready and worker_poller_ready and traps_ready: + if secrets_ready and worker_poller_ready and traps_ready and worker_discovery_ready: # If all three files were loaded into dictionary and updated successfully, # save the latest configuration to files. with open(os.path.join(path_to_compose_files, ".env"), "a") as file: @@ -157,6 +168,23 @@ def load_compose_traps(new_secrets_in_workers, yaml_file): return yaml_file, traps_ready +def load_compose_worker_discovery(new_secrets_in_workers, yaml_file): + # If the secret should be added to worker discovery, load docker-compose-worker-discovery.yaml to a dictionary and + # update "secrets" section. + try: + if "secrets" not in yaml_file["services"]["worker-discovery"]: + yaml_file["services"]["worker-discovery"]["secrets"] = [] + yaml_file["services"]["worker-discovery"]["secrets"].extend( + new_secrets_in_workers + ) + worker_discovery_ready = True + except Exception as e: + print(f"Problem with editing worker-discovery. Secret not added. Error {e}") + yaml_file = {} + worker_discovery_ready = False + return yaml_file, worker_discovery_ready + + def load_compose_worker_poller(new_secrets_in_workers, yaml_file): # If the secret should be added to worker poller, load docker-compose-worker-poller.yaml to a dictionary and # update "secrets" section. @@ -200,6 +228,7 @@ def delete_secrets( secret_name: str, make_change_in_worker_poller: bool, make_change_in_traps: bool, + make_change_in_worker_discovery: bool, ): """ Function to delete secrets from .env and docker-compose.yaml files @@ -208,6 +237,7 @@ def delete_secrets( @param secret_name: name of the secret @param make_change_in_worker_poller: flag indicating whether to delete secrets from worker poller service @param make_change_in_traps: flag indicating whether to delete secrets from traps service + @param make_change_in_worker_discovery: flag indicating whether to delete secrets from worker discovery service """ secrets = [] for key in variables.keys(): @@ -240,6 +270,16 @@ def delete_secrets( ) ) + if make_change_in_worker_discovery: + # filter out secrets destined for deletion + + yaml_file["services"]["worker-discovery"]["secrets"] = list( + filter( + lambda el: el["source"] not in secrets, + yaml_file["services"]["worker-discovery"]["secrets"], + ) + ) + except Exception as e: print(f"Problem with editing secrets section. Secret not added. Error: {e}") @@ -290,6 +330,9 @@ def main(): parser.add_argument( "--worker_poller", default="true", help="Add secret to worker poller" ) + parser.add_argument( + "--worker_discovery", default="true", help="Add secret to worker discovery" + ) parser.add_argument("--traps", default="true", help="Add secret to traps") parser.add_argument("--userName", default="", help="SNMPV3 username") parser.add_argument("--privProtocol", default="", help="SNMPV3 privProtocol") @@ -306,6 +349,7 @@ def main(): path_to_compose_files = args.path_to_compose make_change_in_worker_poller = human_bool(args.worker_poller) make_change_in_traps = human_bool(args.traps) + make_change_in_worker_discovery = human_bool(args.worker_discovery) # variables dictionary maps variables names stored inside a secret to their values variables = { @@ -332,6 +376,7 @@ def main(): secret_name, make_change_in_worker_poller, make_change_in_traps, + make_change_in_worker_discovery, ) except ValueError as e: print(e) @@ -342,6 +387,7 @@ def main(): secret_name, make_change_in_worker_poller, make_change_in_traps, + make_change_in_worker_discovery, ) diff --git a/docs/architecture/design.md b/docs/architecture/design.md index 3a5bee804..6706cb207 100644 --- a/docs/architecture/design.md +++ b/docs/architecture/design.md @@ -25,6 +25,7 @@ Diagram above present high level architecture of Splunk Connector for SNMP, it c - **Trap** - responsible for listening and receiving trap notifications from SNMP agents. The listener is always waiting for the messages coming on the specified port and passing them to the trap worker for further processing. +- **Discovery** - responsible for detecting SNMP-enabled devices within a given subnet. Celery is used to schedule and execute the discovery tasks, with Redis acting as the message broker. - **MIB Server** - responsible for serving MIBs to SNMP Workers and translating oids to varbinds. - **MongoDB** - used for storing configuration and state of the SC4SNMP. - **Inventory** - job used for updating the information about SC4SNMP configuration. It is run after every update to diff --git a/docs/dashboard.md b/docs/dashboard.md index 15f35ae70..182deaf5f 100644 --- a/docs/dashboard.md +++ b/docs/dashboard.md @@ -73,6 +73,18 @@ otherwise we will see information with another status. ![Trap dashboards](images/dashboard/trap_dashboard.png) +#### Discovery dashboards + +To check that discovery for your discovery key is working correctly, look at **SNMP schedule of discovery tasks** dashboard. +With this chart you can understand when SC4SNMP scheduled discovery for your discovery key last time. The process works if it runs regularly. + +After double-checking that SC4SNMP scheduled discovery tasks for your discovery key we need to be sure that discovery is working. +For that look at another dashboard **SNMP discovery status** and if everything is working you will see only **succeeded** status of discovery. +If something is going wrong you will see another status (like on screenshot), then use [troubleshooting docs +for that](troubleshooting/discovery-issues.md). + +![Discovery dashboards](images/dashboard/discovery_dashboard.png) + #### Other dashboards We also have tasks that will be a callback for walk and poll. For example **send** will publish result in Splunk. diff --git a/docs/discovery.md b/docs/discovery.md new file mode 100644 index 000000000..5444b6077 --- /dev/null +++ b/docs/discovery.md @@ -0,0 +1,51 @@ +# Discovery + +## Purpose +The SNMP Discovery feature in Splunk Connect for SNMP provides an automated way to identify SNMP-enabled devices within user-specified subnets. Instead of manually scanning networks or maintaining a static list of devices, users can now use this feature to generate an up-to-date list of IP addresses where SNMP is actively running. + +!!! info + The current implementation does not automatically integrate discovered devices into the polling pipeline. The discovered IPs are saved to an output file, which can then be reviewed or used manually to update your SNMP polling configuration. + + +### This feature is useful when: +- Visibility into which devices in the network have SNMP enabled is required. +- A fast way to generate a list of devices for which further monitoring is needed. + +## How It Works +The discovery process consists of two main steps: + +### 1. List devices +To begin, the system identifies all the devices within the defined subnet. + +### 2. SNMP Probing +Once the list of devices is identified: + +- The system sends SNMP requests to each device using the credentials specified in the configuration (e.g., community strings or SNMPv3 secrets). +- If the device responds successfully to an SNMP poll, the IP is considered SNMP-enabled. +- All such devices along with some details are saved to a defined output file. + +This output can later be used by the user to configure polling. + +### Multi-Subnet Support +Multiple discovery jobs can be configured to run independently for different subnets. Each job can have its frequency, SNMP version, credentials, and grouping logic. This makes it easy to scan different parts of your network separately. + +## Output Format +After each discovery run, a file named `discovery_devices.csv` is generated in the path defined by `discoveryPath`. This file includes all successfully discovered SNMP devices and can be used in poller configuration. The CSV file contains fields like key (discovery name), subnet, ip address, port, snmp version, group, secret, and community. + +To use this feature, the user must provide a valid path where the CSV file will be created. Note that this is a single shared file, and all discovery jobs for different subnets will update the same file. + +Example: + +```csv +key,subnet,ip,port,version,group,secret,community +discovery_version2c,10.202.4.200/30,10.202.4.202,161,2c,linux-group,,public +``` + +!!! info + This file serves as a reference and does not automatically update any active polling configuration. Users are expected to manually review and incorporate this list into their SNMP polling setup as needed. + + +## Configuration +To configure and run SNMP Autodiscovery, refer to: +- [Docker Compose discovery configuration](./dockercompose/11-discovery-configuration.md) +- [Microk8s discovery configuration](./microk8s/configuration/discovery-configuration.md) diff --git a/docs/dockercompose/11-discovery-configuration.md b/docs/dockercompose/11-discovery-configuration.md new file mode 100644 index 000000000..11cc917ec --- /dev/null +++ b/docs/dockercompose/11-discovery-configuration.md @@ -0,0 +1,63 @@ +# Discovery configuration + +Discovery configuration is stored in the `discovery-config.yaml` file. This file has the following sections: + +```yaml +enabled: +ipv6Enabled: +autodiscovery: + discovery_key: + frequency: + delete_already_discovered: + network_address: + version: + community: + port: + device_rules: + - name: + patterns: + Group: + +``` + +- `enabled`: To enable or disable the discovery feature set `enabled` key. The default value is `false`. +- `ipv6Enabled`: To enable IPv6 subnet scanning set `ipv6Enabled` key to be `true`. + +!!! info + If `ipv6Enabled` is `false`, then the task will not be created for discovery key with IPv6 network address. + +- `autodiscovery`: Discovery tasks are defined under the autodiscovery section. Each discovery task can target a specific subnet with its own SNMP version and settings. +Discovery key (i.e. task name) must start with a letter (not a number). Configuration of this section looks the same as in the `values.yaml` in `discovery.autodiscovery` section, which can be checked in the documentation of [discovery configuration](../microk8s/configuration/discovery-configuration.md). + +## Example of the configuration + +```yaml +enabled: true +ipv6Enabled: true +autodiscovery: + discovery_version2c: + frequency: 86400 + delete_already_discovered: true + network_address: 10.202.4.200/30 + version: "2c" + community: "public" + port: 161 + device_rules: + - name: "Linux servers" + patterns: "*linux*" + group: "linux-group" + + discovery_version3: + frequency: 43200 + delete_already_discovered: false + network_address: 10.202.4.200/30 + version: "3" + port: 161 + secret: sc4snmp-hlab-sha-aes + security_engine: "80001f8880e761866965756b6800000000" + device_rules: + - name: "Windows servers" + patterns: "*Windows*" + group: "windows-group" + +``` diff --git a/docs/dockercompose/2-download-package.md b/docs/dockercompose/2-download-package.md index 6a00101aa..5e1dfb2ef 100644 --- a/docs/dockercompose/2-download-package.md +++ b/docs/dockercompose/2-download-package.md @@ -5,7 +5,7 @@ Package with docker compose configuration files (`docker_compose.zip`) can be do ## Configuration To configure the deployment, follow the instructions in [Inventory configuration](./3-inventory-configuration.md), -[Scheduler configuration](./4-scheduler-configuration.md), [Traps configuration](./5-traps-configuration.md), +[Scheduler configuration](./4-scheduler-configuration.md), [Traps configuration](./5-traps-configuration.md), [Discovery configuration](./11-discovery-configuration.md), [.env file configuration](./6-env-file-configuration.md), [SNMPv3 secrets](./7-snmpv3-secrets.md). ## Deploying the app diff --git a/docs/dockercompose/6-env-file-configuration.md b/docs/dockercompose/6-env-file-configuration.md index ce180b4ba..4fb65c1d7 100644 --- a/docs/dockercompose/6-env-file-configuration.md +++ b/docs/dockercompose/6-env-file-configuration.md @@ -11,6 +11,7 @@ Inside the directory with the docker compose files, there is a `.env`. Variables | `SCHEDULER_CONFIG_FILE_ABSOLUTE_PATH` | Absolute path to [scheduler-config.yaml](./4-scheduler-configuration.md) file | | `TRAPS_CONFIG_FILE_ABSOLUTE_PATH` | Absolute path to [traps-config.yaml](./5-traps-configuration.md) file | | `INVENTORY_FILE_ABSOLUTE_PATH` | Absolute path to [inventory.csv](./3-inventory-configuration.md) file | +| `DISCOVERY_CONFIG_FILE_ABSOLUTE_PATH` | Absolute path to [discovery-config.yaml](./11-discovery-configuration.md) file | | `COREFILE_ABS_PATH` | Absolute path to Corefile used by coreDNS. Default Corefile can be found inside the `docker_compose` | | `SC4SNMP_VERSION` | Version of SC4SNMP | @@ -66,17 +67,18 @@ Inside the directory with the docker compose files, there is a `.env`. Variables ## Workers ### General -| Variable | Description | -|------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------| -| `WALK_RETRY_MAX_INTERVAL` | Maximum time interval between walk attempts | -| `WALK_MAX_RETRIES` | Maximum number of walk retries | -| `METRICS_INDEXING_ENABLED` | Details can be found in [append oid index part to the metrics](../microk8s/configuration/poller-configuration.md#append-oid-index-part-to-the-metrics) | -| `POLL_BASE_PROFILES` | Enable polling base profiles (with IF-MIB and SNMPv2-MIB) | -| `IGNORE_NOT_INCREASING_OIDS` | Ignoring `occurred: OID not increasing` issues for hosts specified in the array, ex: IGNORE_NOT_INCREASING_OIDS=127.0.0.1:164,127.0.0.6 | -| `WORKER_LOG_LEVEL` | Logging level of the workers, possible options: DEBUG, INFO, WARNING, ERROR, CRITICAL, or FATAL | -| `UDP_CONNECTION_TIMEOUT` | Timeout in seconds for SNMP operations | -| `MAX_OID_TO_PROCESS` | Sometimes SNMP Agent cannot accept more than X OIDs per once, so if the error "TooBig" is visible in logs, decrease the number of MAX_OID_TO_PROCESS | -| `MAX_REPETITIONS` | The amount of requested next oids in response for each of varbinds in one request sent | +| Variable | Description | +|------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------| +| `WALK_RETRY_MAX_INTERVAL` | Maximum time interval between walk attempts | +| `WALK_MAX_RETRIES` | Maximum number of walk retries | +| `METRICS_INDEXING_ENABLED` | Details can be found in [append oid index part to the metrics](../microk8s/configuration/poller-configuration.md#append-oid-index-part-to-the-metrics) | +| `POLL_BASE_PROFILES` | Enable polling base profiles (with IF-MIB and SNMPv2-MIB) | +| `IGNORE_NOT_INCREASING_OIDS` | Ignoring `occurred: OID not increasing` issues for hosts specified in the array, ex: IGNORE_NOT_INCREASING_OIDS=127.0.0.1:164,127.0.0.6 | +| `WORKER_LOG_LEVEL` | Logging level of the workers, possible options: DEBUG, INFO, WARNING, ERROR, CRITICAL, or FATAL | +| `UDP_CONNECTION_TIMEOUT` | Timeout in seconds for SNMP operations | +| `UDP_CONNECTION_RETRIES` | Number of retries for SNMP operations | +| `CELERY_TASK_TIMEOUT` | Timeout in seconds for celery task | +| `MAX_OID_TO_PROCESS` | Sometimes SNMP Agent cannot accept more than X OIDs per once, so if the error "TooBig" is visible in logs, decrease the number of MAX_OID_TO_PROCESS | ### Worker Poller | Variable | Description | @@ -114,6 +116,17 @@ Inside the directory with the docker compose files, there is a `.env`. Variables | `WORKER_TRAP_CPU_RESERVATIONS` | Dedicated cpu resources for worker trap container | | `WORKER_TRAP_MEMORY_RESERVATIONS` | Dedicated memory resources for worker trap container | +### Worker Discovery +| Variable | Description | +|----------------------------------------|-------------------------------------------------------------------------------| +| `WORKER_DISCOVERY_CONCURRENCY` | Minimum number of threads in the discovery container | +| `PREFETCH_DISCOVERY_COUNT` | How many tasks are consumed from the queue at once in the discovery container | +| `WORKER_DISCOVERY_REPLICAS` | Number of docker replicas of worker discovery container | +| `WORKER_DISCOVERY_CPU_LIMIT` | Limit of cpu that worker discovery container can use | +| `WORKER_DISCOVERY_MEMORY_LIMIT` | Limit of memory that worker discovery container can use | +| `WORKER_DISCOVERY_CPU_RESERVATIONS` | Dedicated cpu resources for worker discovery container | +| `WORKER_DISCOVERY_MEMORY_RESERVATIONS` | Dedicated memory resources for worker discovery container | + ## Inventory | Variable | Description | @@ -128,8 +141,17 @@ Inside the directory with the docker compose files, there is a `.env`. Variables | `SNMP_V3_SECURITY_ENGINE_ID` | SNMPv3 TRAPs require the configuration SNMP Engine ID of the TRAP sending application for the USM users table of the TRAP receiving application for each USM user, for example: SNMP_V3_SECURITY_ENGINE_ID=80003a8c04,aab123456 | | `INCLUDE_SECURITY_CONTEXT_ID` | Controls whether to add the context_engine_id field to v3 trap events | | `TRAPS_PORT` | External port exposed for traps server | + ## Scheduler | Variable | Description | |-----------------------|---------------------------------------------------------------------------------------------------| -| `SCHEDULER_LOG_LEVEL` | Logging level of the scheduler, possible options: DEBUG, INFO, WARNING, ERROR, CRITICAL, or FATAL | \ No newline at end of file +| `SCHEDULER_LOG_LEVEL` | Logging level of the scheduler, possible options: DEBUG, INFO, WARNING, ERROR, CRITICAL, or FATAL | + +## Discovery + +| Variable | Description | +|-----------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `DISCOVERY_ENABLE` | Enable for discovery feature | +| `DISCOVERY_LOG_LEVEL` | Logging level of the discovery, possible options: DEBUG, INFO, WARNING, ERROR, CRITICAL, or FATAL | +| `DISCOVERY_PATH` | It specifies the absolute path on the local file system where the `discovery_devices.csv` file will be created. If the CSV file is not already present then new file will be created. | diff --git a/docs/dockercompose/7-snmpv3-secrets.md b/docs/dockercompose/7-snmpv3-secrets.md index e29e8ed65..5c7c770bc 100644 --- a/docs/dockercompose/7-snmpv3-secrets.md +++ b/docs/dockercompose/7-snmpv3-secrets.md @@ -14,18 +14,19 @@ pip3 install ruamel.yaml To create a new secret, `manage_secrets.py` must be run with the following flags: -| Flag | Description | -|---------------------|--------------------------------------------------------------------------------| -| `--secret_name` | New secret name | -| `--path_to_compose` | Absolute path to directory with docker compose files | -| `--worker_poller` | \[OPTIONAL\] Add new secrets to worker poller. Default value is set to 'true'. | -| `--traps` | \[OPTIONAL\] Add new secrets to traps server. Default value is set to 'true'. | -| `--userName` | SNMPv3 userName | -| `--privProtocol` | SNMPv3 privProtocol | -| `--privKey` | SNMPv3 privKey | -| `--authProtocol` | SNMPv3 authProtocol | -| `--authKey` | SNMPv3 authKey | -| `--contextEngineId` | \[OPTIONAL\] SNMPv3 engine id | +| Flag | Description | +|---------------------|-----------------------------------------------------------------------------------| +| `--secret_name` | New secret name | +| `--path_to_compose` | Absolute path to directory with docker compose files | +| `--worker_poller` | \[OPTIONAL\] Add new secrets to worker poller. Default value is set to 'true'. | +| `--worker_discovery`| \[OPTIONAL\] Add new secrets to worker discovery. Default value is set to 'true'. | +| `--traps` | \[OPTIONAL\] Add new secrets to traps server. Default value is set to 'true'. | +| `--userName` | SNMPv3 userName | +| `--privProtocol` | SNMPv3 privProtocol | +| `--privKey` | SNMPv3 privKey | +| `--authProtocol` | SNMPv3 authProtocol | +| `--authKey` | SNMPv3 authKey | +| `--contextEngineId` | \[OPTIONAL\] SNMPv3 engine id | This script, apart from updating configuration files, creates environmental variables with values of the secret at the diff --git a/docs/images/dashboard/discovery_dashboard.png b/docs/images/dashboard/discovery_dashboard.png new file mode 100644 index 000000000..bf9b735da Binary files /dev/null and b/docs/images/dashboard/discovery_dashboard.png differ diff --git a/docs/images/sc4snmp_architecture.png b/docs/images/sc4snmp_architecture.png index a79189d28..56b7e9ebf 100644 Binary files a/docs/images/sc4snmp_architecture.png and b/docs/images/sc4snmp_architecture.png differ diff --git a/docs/microk8s/configuration/deployment-configuration.md b/docs/microk8s/configuration/deployment-configuration.md index b04de7d6e..beb91c5f9 100644 --- a/docs/microk8s/configuration/deployment-configuration.md +++ b/docs/microk8s/configuration/deployment-configuration.md @@ -25,6 +25,10 @@ For traps receiving purposes: - `traps` - For more details see [trap configuration](trap-configuration.md). +For SNMP devices discovery purpose: + +- `discovery` - For more details see [Discovery configuration](discovery-configuration.md). + Shared components: - `inventory` - For more details see [inventory configuration](../poller-configuration#configure-inventory). diff --git a/docs/microk8s/configuration/discovery-configuration.md b/docs/microk8s/configuration/discovery-configuration.md new file mode 100644 index 000000000..3c52d3635 --- /dev/null +++ b/docs/microk8s/configuration/discovery-configuration.md @@ -0,0 +1,155 @@ +# Discovery Configuration + +The discovery feature automatically discovers SNMP-enabled devices within a given subnet. Based on the discovery results, a `discovery_devices.csv` is generated and can be used to configure polling. + +Discovery supports IPv4 and IPv6 subnets, SNMP v1, v2c, and v3 devices, and basic grouping of devices using SNMP `sysDescr` from `SNMPv2-MIB` (OID `1.3.6.1.2.1.1.1.0`). + + +### Discovery configuration file + +The discovery configuration is kept in the `values.yaml` file in the discovery section. +`values.yaml` is used during the installation process for configuring Kubernetes values. + +See the following discovery example configuration: +```yaml +discovery: + enabled: true + logLevel: "DEBUG" + ipv6Enabled: true + discoveryPath: "/home/user/sc4snmp" + usernameSecrets: + - sc4snmp-hlab-sha-aes + + autodiscovery: + discovery_version2c: + frequency: 86400 + delete_already_discovered: true + network_address: 10.202.4.200/30 + version: "2c" + community: "public" + port: 161 + device_rules: + - name: "Linux servers" + patterns: "*linux*" + group: "linux-group" + + discovery_version3: + frequency: 43200 + delete_already_discovered: false + network_address: 10.202.4.200/30 + version: "3" + port: 161 + secret: sc4snmp-hlab-sha-aes + security_engine: "80001f8880e761866965756b6800000000" + device_rules: + - name: "Windows servers" + patterns: "*Windows*" + group: "windows-group" + +``` + +### Enable Discovery +To enable or disable the discovery feature set `enabled` key. +The default value is `false`. + +### Define log level +The log level for discovery can be set by changing the value for the `logLevel` key. The allowed values are`DEBUG`, `INFO`, `WARNING`, or `ERROR`. +The default value is `WARNING`. + +### Enable IPv6 +To enable IPv6 subnet scanning, set `ipv6Enabled` key. + +!!! info + If `ipv6Enabled` is `false`, then the task will not be created for discovery key with IPv6 network address. + +### Define Discovery Path +`discoveryPath` specifies the absolute path on the local file system where the `discovery_devices.csv` file will be created. +If the CSV file is not already present, then a new file will be created. + +!!! info + The path provided should have read-write permission for user and group `10001`. + +### Define usernamesecrets +The `usernameSecrets` key in the `discovery` section defines the SNMPv3 secrets for the discovery of the SNMP device. +`usernameSecrets` defines which secrets in "Secret" objects in k8s should be used, as a value, it needs the name of "Secret" objects. +For more information on how to define the "Secret" object for SNMPv3, see [SNMPv3 Configuration](snmpv3-configuration.md). + +See the following example: +```yaml +discovery: + usernameSecrets: + - sc4snmp-homesecure-sha-aes + - sc4snmp-homesecure-sha-des +``` + +### Configure discovery tasks +Discovery tasks are defined under the autodiscovery section. Each discovery task can target a specific subnet with its own SNMP version and settings. +Discovery key (i.e. task name) must start with a letter (not a number). + +Each task has the following fields to configure: + +| Field | Description | Default | Required | +|------------------ |-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------|----------| +| `frequency` | Time interval (in minutes) between each run of the discovery task. Note: If the frequency is less than 6 hours, it will be taken as 6 hours by default. | `86400` | NO | +| `delete_already_discovered` | Deletes old entries of a particular discovery key before writing new ones. | `false` | NO | +| `network_address` | Subnet in CIDR notation to scan. Supports IPv4 or IPv6. | | YES | +| `port` | SNMP listening port. | `161` | NO | +| `version` | SNMP version, the allowed values are `1`, `2c`, or `3`. | `2c` | NO | +| `community` | SNMP community string, this field is required when the `version` is `1` or `2c`. | | NO | +| `secret` | The reference to the secret from `discovery.usernameSecrets` that should be used to discover, this field is required when the `version` is `3` devices. | | NO | +| `security_engine` | The security engine ID required by SNMPv3. If it is not provided for version `3`, it will be autogenerated. | | NO | + + +### Define delete_already_discovered +The `delete_already_discovered` flag controls whether devices found in previous discovery runs are kept. + +Since the discovery task runs at fixed intervals to scan for SNMP-enabled devices: + - If set to `true`, all devices discovered in the previous run under the same discovery key will be deleted. This is useful when you want to ensure that the list always reflects the most up-to-date set of devices. + - If set to `false`, it will retain devices discovered in earlier runs, and new devices will be appended to the existing list. This is useful when you want to keep a cumulative list of all SNMP-enabled devices discovered over time. + +### Define device_rules +The `device_rules` section is used to organize discovered devices into logical groups based on pattern matching against their SNMP system descriptions (sysDescr). + +Each rule consists of: + +- `name`: A label to identify the rule. It is used for reference and should be unique within the list. +- `patterns`: A wildcard pattern (supports `*`) that matches against the `sysDescr` returned from SNMP. +- `group`: The name of the group to assign the matched devices to. This group can later be referenced for polling or other configurations. + +**Example** +```yaml +device_rules: + - name: "Linux Devices" + patterns: "*Linux*" + group: "linux-group" +``` + +### Configure Timeouts and Retries + +**Example** +```yaml +worker: + taskTimeout: 8000 + udpConnectionTimeout: 3 + udpConnectionRetries: 5 +``` + +The following fields help control how long discovery tasks run and how SNMP responses are handled, especially for slower networks or larger subnets: + +#### `taskTimeout` + +Defines the **maximum execution time (in seconds)** for a single discovery task. +- Default: `2400` seconds. +- Increase this if you are scanning large subnets or using longer SNMP retry configurations. + +Make sure `taskTimeout` is large enough to accommodate the `nmap` scan and the SNMP checks across all IPs. + +#### `udpConnectionTimeout` + +Specifies the **timeout (in seconds)** for each SNMP request (`getCmd`). +Increase this if devices take longer to respond or if there is network latency. + +#### `udpConnectionRetries` + +Determines how many times a request is retried if there is no response. +Higher retries can improve success rates on unstable networks, but will increase total execution time. diff --git a/docs/microk8s/configuration/values-params-description.md b/docs/microk8s/configuration/values-params-description.md index 0fa4f0a74..aee1163b5 100644 --- a/docs/microk8s/configuration/values-params-description.md +++ b/docs/microk8s/configuration/values-params-description.md @@ -120,7 +120,8 @@ Detailed documentation about configuring worker can be found in [Worker](worker- | `poller` | Section with configuration for worker poller pods | | | `trap` | Section with configuration for worker trap pods | | | `sender` | Section with configuration for worker sender pods | | -| `x.replicaCount` | Number of pod replicas when autoscaling is disabled | poller/trap - `2`, sender - `1` | +| `discovery` | Section with configuration for worker discovery pods | | +| `x.replicaCount` | Number of pod replicas when autoscaling is disabled | poller/trap - `2`, discovery/sender - `1` | | `x.concurrency` | Minimum number of threads in a pod | `4` | | `x.prefetch` | Number of tasks consumed from the queue at once | poller - `1`, traps/sender - `30` | | `x.autoscaling.enabled` | Enables autoscaling for pod | poller - `false` | @@ -143,6 +144,7 @@ Detailed documentation about configuring worker can be found in [Worker](worker- | `profilesReloadDelay` | Delay of polling profiles after inventory reload | `60` | | `logLevel` | Log level for workers | `INFO` | | `udpConnectionTimeout` | Timeout for SNMP operations in seconds | `3` | +| `udpConnectionRetries` | Number of retries for SNMP operations | `5` | | `ignoreEmptyVarbinds` | Ignores "Empty SNMP response message" in responses | `false` | | `podAntiAffinity` | [Kubernetes documentation](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity) | `soft` | | `nodeSelector` | [Kubernetes documentation](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector) | | @@ -194,6 +196,29 @@ Detailed documentation about configuring traps can be found in [Traps](trap-conf | `nodeSelector` | [Kubernetes documentation](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector) | | | `tolerations` | [Kubernetes documentation](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/) | | +## Discovery + +Detailed documentation about configuring discovery can be found in [Discovery](discovery-configuration.md). + +| Variable | Description | Default | +|-------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------|------------------| +| `enabled` | Enables discovering SNMP-enabled devices and creates a CSV file. | `false` | +| `usernameSecrets` | Defines SNMPv3 secrets for trap messages sent by SNMP device | | +| `ipv6Enabled` | Enabled device detection using IPv6 subnet | `false` | +| `logLevel` | Log level for a discovery pod | `INFO` | +| `discoveryPath` | Absolute path where discovery_devices.csv will be stored | | +| `autodiscovery.x.frequency` | Time interval (in minutes) between each run of the discovery task | `86400` | +| `autodiscovery.x.delete_already_discovered` | Deletes old entries of a particular discovery key before writing new ones. | `false` | +| `autodiscovery.x.network_address` | Subnet in CIDR notation to scan. | | +| `autodiscovery.x.version` | SNMP version to use | `2c` | +| `autodiscovery.x.community` | SNMP community string | | +| `autodiscovery.x.port` | SNMP port to use | `161` | +| `autodiscovery.x.secret` | Name of existing secret in kubernetes | | +| `autodiscovery.x.security_engine` | SNMP Engine ID | | +| `autodiscovery.x.device_rules.name` | Device Rule name for reference | | +| `autodiscovery.x.device_rules.patterns` | Wildcard pattern to match SNMP sysDescr | | +| `autodiscovery.x.device_rules.group` | Group name to assign matched devices | | + ## serviceAccount | Variable | Description | Default | diff --git a/docs/microk8s/configuration/worker-configuration.md b/docs/microk8s/configuration/worker-configuration.md index 7e98419c7..cca2ddf21 100644 --- a/docs/microk8s/configuration/worker-configuration.md +++ b/docs/microk8s/configuration/worker-configuration.md @@ -12,6 +12,10 @@ SC4SNMP has two base functionalities: monitoring traps and polling. These operat 3. The `sender` worker handles sending data to Splunk. You need to always have at least one sender pod running. +SC4SNMP also has a discovery functionality which is handled by the worker below: + +1. The `discovery` worker consumes all the tasks related to discovery. + ### Worker configuration file Worker configuration is kept in the `values.yaml` file in the `worker` section. `worker` has 3 subsections: `poller`, `sender`, and `trap`, that refer to the workers types. @@ -102,6 +106,30 @@ worker: # the resources requests for sender worker container requests: cpu: 250m + # The discovery worker handles auto discovery of SNMP-enabled devices and creates a CSV file for it + discovery: + # number of the discovery replicas when autoscaling is set to false + replicaCount: 1 + # minimum number of threads in a pod + concurrency: 4 + # how many tasks are consumed from the queue at once + prefetch: 30 + autoscaling: + # enabling autoscaling for discovery worker pods + enabled: false + # minimum number of running discovery worker pods when autoscaling is enabled + minReplicas: 2 + # maximum number of running discovery worker pods when autoscaling is enabled + maxReplicas: 10 + # CPU % threshold that must be exceeded on discovery worker pods to spawn another replica + targetCPUUtilizationPercentage: 80 + resources: + # the resources limits for discovery worker container + limits: + cpu: 500m + # the resources requests for discovery worker container + requests: + cpu: 250m # Liveness probes are used in Kubernetes to know when a pod is alive or dead. # A pod can be in a dead state for a number of reasons; # the application could be crashed, some error in the application etc. @@ -154,6 +182,8 @@ worker: podAntiAffinity: soft # udpConnectionTimeout timeout in seconds for SNMP operations udpConnectionTimeout: 3 + # udpConnectionRetries number of retries for SNMP operations + udpConnectionRetries: 5 # in case of seeing "Empty SNMP response message" this variable can be set to true ignoreEmptyVarbinds: false @@ -184,6 +214,8 @@ worker: replicaCount: 1 poller: replicaCount: 0 + discovery: + replicaCount: 0 logLevel: "WARNING" ``` @@ -205,6 +237,8 @@ worker: targetCPUUtilizationPercentage: 80 poller: replicaCount: 0 + discovery: + replicaCount: 0 logLevel: "WARNING" ``` @@ -255,6 +289,8 @@ worker: minReplicas: 2 maxReplicas: 20 targetCPUUtilizationPercentage: 80 + discovery: + replicaCount: 0 logLevel: "WARNING" ``` @@ -325,52 +361,62 @@ Trap worker uses in memory cache to store the results of the reverse dns lookup. ### Worker parameters -| Variable | Description | Default | -|----------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------|-------------------| -| worker.poller.replicaCount | Number of poller worker replicas | 2 | -| worker.poller.concurrency | Minimum number of threads in a poller worker pod | 4 | -| worker.poller.prefetch | Number of tasks consumed from the queue at once | 1 | -| worker.poller.autoscaling.enabled | Enabling autoscaling for poller worker pods | false | -| worker.poller.autoscaling.minReplicas | Minimum number of running poller worker pods when autoscaling is enabled | 2 | -| worker.poller.autoscaling.maxReplicas | Maximum number of running poller worker pods when autoscaling is enabled | 10 | -| worker.poller.autoscaling.targetCPUUtilizationPercentage | CPU % threshold that must be exceeded on poller worker pods to spawn another replica | 80 | -| worker.poller.resources.limits | The resources limits for poller worker container | cpu: 500m | -| worker.poller.resources.requests | The requested resources for poller worker container | cpu: 250m | -| worker.trap.replicaCount | Number of trap worker replicas | 2 | -| worker.trap.concurrency | Minimum number of threads in a trap worker pod | 4 | -| worker.trap.prefetch | Number of tasks consumed from the queue at once | 30 | -| worker.trap.resolveAddress.enabled | Enable reverse dns lookup of the IP address of the processed trap | false | -| worker.trap.resolveAddress.cacheSize | Maximum number of reverse dns lookup result records stored in cache | 500 | -| worker.trap.resolveAddress.cacheTTL | Time to live of the cached reverse dns lookup record in seconds | 1800 | -| worker.trap.autoscaling.enabled | Enabling autoscaling for trap worker pods | false | -| worker.trap.autoscaling.minReplicas | Minimum number of running trap worker pods when autoscaling is enabled | 2 | -| worker.trap.autoscaling.maxReplicas | Maximum number of running trap worker pods when autoscaling is enabled | 10 | -| worker.trap.autoscaling.targetCPUUtilizationPercentage | CPU % threshold that must be exceeded on trap worker pods to spawn another replica | 80 | -| worker.trap.resources.limits | The resource limit for the poller worker container | cpu: 500m | -| worker.trap.resources.requests | The requested resources for the poller worker container | cpu: 250m | -| worker.sender.replicaCount | The number of sender worker replicas | 1 | -| worker.sender.concurrency | Minimum number of threads in a sender worker pod | 4 | -| worker.sender.prefetch | Number of tasks consumed from the queue at once | 30 | -| worker.sender.autoscaling.enabled | Enabling autoscaling for sender worker pods | false | -| worker.sender.autoscaling.minReplicas | Minimum number of running sender worker pods when autoscaling is enabled | 2 | -| worker.sender.autoscaling.maxReplicas | Maximum number of running sender worker pods when autoscaling is enabled | 10 | -| worker.sender.autoscaling.targetCPUUtilizationPercentage | CPU % threshold that must be exceeded on sender worker pods to spawn another replica | 80 | -| worker.sender.resources.limits | The resource limit for the poller worker container | cpu: 500m | -| worker.sender.resources.requests | The requested resources for the poller worker container | cpu: 250m | -| worker.livenessProbe.enabled | Whether the liveness probe is enabled | false | -| worker.livenessProbe.exec.command | The exec command for the liveness probe to run in the container | Check values.yaml | -| worker.livenessProbe.initialDelaySeconds | Number of seconds after the container has started before liveness probe is initiated | 80 | -| worker.livenessProbe.periodSeconds | Frequency of performing the probe in seconds | 10 | -| worker.readinessProbe.enabled | Whether the readiness probe should be turned on or not | false | -| worker.readinessProbe.exec.command | The exec command for the readiness probe to run in the container | Check values.yaml | -| worker.readinessProbe.initialDelaySeconds | Number of seconds after the container has started before readiness probe is initiated | 30 | -| worker.readinessProbe.periodSeconds | Frequency of performing the probe in seconds | 5 | -| worker.taskTimeout | Task timeout in seconds when process takes a long time | 2400 | -| worker.walkRetryMaxInterval | Maximum time interval between walk attempts | 180 | -| worker.walkMaxRetries | Maximum number of walk retries | 5 | -| worker.ignoreNotIncreasingOid | Ignoring `occurred: OID not increasing` issues for hosts specified in the array | [] | -| worker.logLevel | Logging level, possible options: DEBUG, INFO, WARNING, ERROR, CRITICAL, or FATAL | INFO | +| Variable | Description | Default | +|-------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------|-------------------| +| worker.poller.replicaCount | Number of poller worker replicas | 2 | +| worker.poller.concurrency | Minimum number of threads in a poller worker pod | 4 | +| worker.poller.prefetch | Number of tasks consumed from the queue at once | 1 | +| worker.poller.autoscaling.enabled | Enabling autoscaling for poller worker pods | false | +| worker.poller.autoscaling.minReplicas | Minimum number of running poller worker pods when autoscaling is enabled | 2 | +| worker.poller.autoscaling.maxReplicas | Maximum number of running poller worker pods when autoscaling is enabled | 10 | +| worker.poller.autoscaling.targetCPUUtilizationPercentage | CPU % threshold that must be exceeded on poller worker pods to spawn another replica | 80 | +| worker.poller.resources.limits | The resources limits for poller worker container | cpu: 500m | +| worker.poller.resources.requests | The requested resources for poller worker container | cpu: 250m | +| worker.trap.replicaCount | Number of trap worker replicas | 2 | +| worker.trap.concurrency | Minimum number of threads in a trap worker pod | 4 | +| worker.trap.prefetch | Number of tasks consumed from the queue at once | 30 | +| worker.trap.resolveAddress.enabled | Enable reverse dns lookup of the IP address of the processed trap | false | +| worker.trap.resolveAddress.cacheSize | Maximum number of reverse dns lookup result records stored in cache | 500 | +| worker.trap.resolveAddress.cacheTTL | Time to live of the cached reverse dns lookup record in seconds | 1800 | +| worker.trap.autoscaling.enabled | Enabling autoscaling for trap worker pods | false | +| worker.trap.autoscaling.minReplicas | Minimum number of running trap worker pods when autoscaling is enabled | 2 | +| worker.trap.autoscaling.maxReplicas | Maximum number of running trap worker pods when autoscaling is enabled | 10 | +| worker.trap.autoscaling.targetCPUUtilizationPercentage | CPU % threshold that must be exceeded on trap worker pods to spawn another replica | 80 | +| worker.trap.resources.limits | The resource limit for the poller worker container | cpu: 500m | +| worker.trap.resources.requests | The requested resources for the poller worker container | cpu: 250m | +| worker.sender.replicaCount | The number of sender worker replicas | 1 | +| worker.sender.concurrency | Minimum number of threads in a sender worker pod | 4 | +| worker.sender.prefetch | Number of tasks consumed from the queue at once | 30 | +| worker.sender.autoscaling.enabled | Enabling autoscaling for sender worker pods | false | +| worker.sender.autoscaling.minReplicas | Minimum number of running sender worker pods when autoscaling is enabled | 2 | +| worker.sender.autoscaling.maxReplicas | Maximum number of running sender worker pods when autoscaling is enabled | 10 | +| worker.sender.autoscaling.targetCPUUtilizationPercentage | CPU % threshold that must be exceeded on sender worker pods to spawn another replica | 80 | +| worker.sender.resources.limits | The resource limit for the poller worker container | cpu: 500m | +| worker.sender.resources.requests | The requested resources for the poller worker container | cpu: 250m | +| worker.discovery.replicaCount | Number of discovery worker replicas | 1 | +| worker.discovery.concurrency | Minimum number of threads in a discovery worker pod | 4 | +| worker.discovery.prefetch | Number of tasks consumed from the queue at once | 30 | +| worker.discovery.autoscaling.enabled | Enabling autoscaling for discovery worker pods | false | +| worker.discovery.autoscaling.minReplicas | Minimum number of running discovery worker pods when autoscaling is enabled | 2 | +| worker.discovery.autoscaling.maxReplicas | Maximum number of running discovery worker pods when autoscaling is enabled | 10 | +| worker.discovery.autoscaling.targetCPUUtilizationPercentage | CPU % threshold that must be exceeded on discovery worker pods to spawn another replica | 80 | +| worker.discovery.resources.limits | The resources limits for discovery worker container | cpu: 500m | +| worker.discovery.resources.requests | The requested resources for discovery worker container | cpu: 250m | +| worker.livenessProbe.enabled | Whether the liveness probe is enabled | false | +| worker.livenessProbe.exec.command | The exec command for the liveness probe to run in the container | Check values.yaml | +| worker.livenessProbe.initialDelaySeconds | Number of seconds after the container has started before liveness probe is initiated | 80 | +| worker.livenessProbe.periodSeconds | Frequency of performing the probe in seconds | 10 | +| worker.readinessProbe.enabled | Whether the readiness probe should be turned on or not | false | +| worker.readinessProbe.exec.command | The exec command for the readiness probe to run in the container | Check values.yaml | +| worker.readinessProbe.initialDelaySeconds | Number of seconds after the container has started before readiness probe is initiated | 30 | +| worker.readinessProbe.periodSeconds | Frequency of performing the probe in seconds | 5 | +| worker.taskTimeout | Task timeout in seconds when process takes a long time | 2400 | +| worker.walkRetryMaxInterval | Maximum time interval between walk attempts | 180 | +| worker.walkMaxRetries | Maximum number of walk retries | 5 | +| worker.ignoreNotIncreasingOid | Ignoring `occurred: OID not increasing` issues for hosts specified in the array | [] | +| worker.logLevel | Logging level, possible options: DEBUG, INFO, WARNING, ERROR, CRITICAL, or FATAL | INFO | | worker.disableMongoDebugLogging | Disable extensive MongoDB and pymongo debug logging on SC4SNMP worker pods | true | -| worker.podAntiAffinity | [Kubernetes documentation](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity) | soft | -| worker.udpConnectionTimeout | Timeout for SNMP operations in seconds | 3 | -| worker.ignoreEmptyVarbinds | Ignores “Empty SNMP response message” in responses | false | +| worker.podAntiAffinity | [Kubernetes documentation](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity) | soft | +| worker.udpConnectionTimeout | Timeout for SNMP operations in seconds | 3 | +| worker.udpConnectionretries | Number of retries for SNMP operations | 5 | +| worker.ignoreEmptyVarbinds | Ignores “Empty SNMP response message” in responses | false | diff --git a/docs/microk8s/enable-ipv6.md b/docs/microk8s/enable-ipv6.md index fdf7d9eea..9784f03f5 100644 --- a/docs/microk8s/enable-ipv6.md +++ b/docs/microk8s/enable-ipv6.md @@ -64,6 +64,13 @@ traps: ipFamilies: ["IPv4", "IPv6"] ``` +To enable SNMP device detection for IPv4 and IPv6 subnet, you need to add the following configuration to `values.yaml` file: + +``` +discovery: + ipv6Enabled: true +``` + To configure poller to poll IPv4 and IPv6 addresses, you need to add the following configuration to the `values.yaml` file: ``` poller: diff --git a/docs/microk8s/mk8s/k8s-microk8s-scaling.md b/docs/microk8s/mk8s/k8s-microk8s-scaling.md index 8efc7a478..4306e5134 100644 --- a/docs/microk8s/mk8s/k8s-microk8s-scaling.md +++ b/docs/microk8s/mk8s/k8s-microk8s-scaling.md @@ -80,6 +80,8 @@ worker: replicaCount: 4 sender: replicaCount: 4 + discovery: + replicaCount: 4 ``` 3. Add `traps` replica count in `values.yaml`: @@ -104,28 +106,32 @@ microk8s kubectl get pods -n sc4snmp You should get 4 replicas for each worker and traps service: ```bash -NAME READY STATUS RESTARTS AGE -snmp-mibserver-5df74fb678-zkj9m 1/1 Running 0 25h -snmp-mongodb-6dc5c4f74d-xg6p7 2/2 Running 0 25h -snmp-redis-master-0 1/1 Running 0 25h -snmp-splunk-connect-for-snmp-inventory-k9t87 0/1 Completed 0 3m -snmp-splunk-connect-for-snmp-scheduler-76848cf748-57qbx 1/1 Running 0 25h -snmp-splunk-connect-for-snmp-trap-9f55664c4-9dv7d 1/1 Running 0 3m1s -snmp-splunk-connect-for-snmp-trap-9f55664c4-crgld 1/1 Running 0 3m1s -snmp-splunk-connect-for-snmp-trap-9f55664c4-sb768 1/1 Running 0 25h -snmp-splunk-connect-for-snmp-trap-9f55664c4-tkhcp 1/1 Running 0 3m1s -snmp-splunk-connect-for-snmp-worker-poller-7487956697-4hvpl 1/1 Running 0 21h -snmp-splunk-connect-for-snmp-worker-poller-7487956697-8bvnn 1/1 Running 0 3m1s -snmp-splunk-connect-for-snmp-worker-poller-7487956697-9dfgt 1/1 Running 0 3m1s -snmp-splunk-connect-for-snmp-worker-poller-7487956697-hlhvz 1/1 Running 0 24h -snmp-splunk-connect-for-snmp-worker-sender-657589666f-979d2 1/1 Running 0 3m1s -snmp-splunk-connect-for-snmp-worker-sender-657589666f-mrvg9 1/1 Running 0 3m1s -snmp-splunk-connect-for-snmp-worker-sender-657589666f-qtcr8 1/1 Running 0 21h -snmp-splunk-connect-for-snmp-worker-sender-657589666f-tc8sv 1/1 Running 0 24h -snmp-splunk-connect-for-snmp-worker-trap-859dc47d9b-6fbs2 1/1 Running 0 24h -snmp-splunk-connect-for-snmp-worker-trap-859dc47d9b-kdcdb 1/1 Running 0 3m1s -snmp-splunk-connect-for-snmp-worker-trap-859dc47d9b-sfxvb 1/1 Running 0 3m -snmp-splunk-connect-for-snmp-worker-trap-859dc47d9b-xmmwv 1/1 Running 0 21h +NAME READY STATUS RESTARTS AGE +snmp-mibserver-5df74fb678-zkj9m 1/1 Running 0 25h +snmp-mongodb-6dc5c4f74d-xg6p7 2/2 Running 0 25h +snmp-redis-master-0 1/1 Running 0 25h +snmp-splunk-connect-for-snmp-inventory-k9t87 0/1 Completed 0 3m +snmp-splunk-connect-for-snmp-scheduler-76848cf748-57qbx 1/1 Running 0 25h +snmp-splunk-connect-for-snmp-trap-9f55664c4-9dv7d 1/1 Running 0 3m1s +snmp-splunk-connect-for-snmp-trap-9f55664c4-crgld 1/1 Running 0 3m1s +snmp-splunk-connect-for-snmp-trap-9f55664c4-sb768 1/1 Running 0 25h +snmp-splunk-connect-for-snmp-trap-9f55664c4-tkhcp 1/1 Running 0 3m1s +snmp-splunk-connect-for-snmp-worker-poller-7487956697-4hvpl 1/1 Running 0 21h +snmp-splunk-connect-for-snmp-worker-poller-7487956697-8bvnn 1/1 Running 0 3m1s +snmp-splunk-connect-for-snmp-worker-poller-7487956697-9dfgt 1/1 Running 0 3m1s +snmp-splunk-connect-for-snmp-worker-poller-7487956697-hlhvz 1/1 Running 0 24h +snmp-splunk-connect-for-snmp-worker-sender-657589666f-979d2 1/1 Running 0 3m1s +snmp-splunk-connect-for-snmp-worker-sender-657589666f-mrvg9 1/1 Running 0 3m1s +snmp-splunk-connect-for-snmp-worker-sender-657589666f-qtcr8 1/1 Running 0 21h +snmp-splunk-connect-for-snmp-worker-sender-657589666f-tc8sv 1/1 Running 0 24h +snmp-splunk-connect-for-snmp-worker-discovery-7d9fdc5d56-js474 1/1 Running 0 3m1s +snmp-splunk-connect-for-snmp-worker-discovery-7d9fdc5d56-j423f 1/1 Running 0 3m1s +snmp-splunk-connect-for-snmp-worker-discovery-7d9fdc5d56-de45b 1/1 Running 0 21h +snmp-splunk-connect-for-snmp-worker-discovery-7d9fdc5d56-8fde5 1/1 Running 0 24h +snmp-splunk-connect-for-snmp-worker-trap-859dc47d9b-6fbs2 1/1 Running 0 24h +snmp-splunk-connect-for-snmp-worker-trap-859dc47d9b-kdcdb 1/1 Running 0 3m1s +snmp-splunk-connect-for-snmp-worker-trap-859dc47d9b-sfxvb 1/1 Running 0 3m +snmp-splunk-connect-for-snmp-worker-trap-859dc47d9b-xmmwv 1/1 Running 0 21h ``` ## Autoscaling SC4SNMP @@ -151,6 +157,11 @@ worker: enabled: true minReplicas: 5 maxReplicas: 10 + discovery: + autoscaling: + enabled: true + minReplicas: 5 + maxReplicas: 10 traps: autoscaling: @@ -174,30 +185,35 @@ microk8s kubectl get po -n sc4snmp After applying the changes, each worker and trap service will have from 5 to 10 instances: ```bash -NAME READY STATUS RESTARTS AGE -snmp-mibserver-6fdcdf9ddd-7bvmj 1/1 Running 0 25h -snmp-mongodb-6dc5c4f74d-6b7mf 2/2 Running 0 25h -snmp-redis-master-0 1/1 Running 0 25h -snmp-splunk-connect-for-snmp-inventory-sssgs 0/1 Completed 0 3m37s -snmp-splunk-connect-for-snmp-scheduler-5fcb6dcb44-r79ff 1/1 Running 0 25h -snmp-splunk-connect-for-snmp-trap-5788bc498c-62xsq 1/1 Running 0 2m10s -snmp-splunk-connect-for-snmp-trap-5788bc498c-bmlhg 1/1 Running 0 2m10s -snmp-splunk-connect-for-snmp-trap-5788bc498c-p7mkq 1/1 Running 0 2m10s -snmp-splunk-connect-for-snmp-trap-5788bc498c-t8q9c 1/1 Running 0 2m10s -snmp-splunk-connect-for-snmp-trap-5788bc498c-xjjp2 1/1 Running 0 24h -snmp-splunk-connect-for-snmp-worker-poller-5d76b9b675-25tbf 1/1 Running 0 16m -snmp-splunk-connect-for-snmp-worker-poller-5d76b9b675-dc6zr 1/1 Running 0 16m -snmp-splunk-connect-for-snmp-worker-poller-5d76b9b675-g7vpr 1/1 Running 0 16m -snmp-splunk-connect-for-snmp-worker-poller-5d76b9b675-gdkgq 1/1 Running 0 16m -snmp-splunk-connect-for-snmp-worker-poller-5d76b9b675-pg6cj 1/1 Running 0 24h -snmp-splunk-connect-for-snmp-worker-sender-7757fb7f89-56h9w 1/1 Running 0 24h -snmp-splunk-connect-for-snmp-worker-sender-7757fb7f89-hr54w 1/1 Running 0 16m -snmp-splunk-connect-for-snmp-worker-sender-7757fb7f89-j7wcn 1/1 Running 0 16m -snmp-splunk-connect-for-snmp-worker-sender-7757fb7f89-sgsdg 0/1 Pending 0 16m -snmp-splunk-connect-for-snmp-worker-sender-7757fb7f89-xrpfx 1/1 Running 0 16m -snmp-splunk-connect-for-snmp-worker-trap-6b8fd89868-79x2l 0/1 Pending 0 16m -snmp-splunk-connect-for-snmp-worker-trap-6b8fd89868-br7pf 1/1 Running 0 24h -snmp-splunk-connect-for-snmp-worker-trap-6b8fd89868-cnmh9 0/1 Pending 0 16m -snmp-splunk-connect-for-snmp-worker-trap-6b8fd89868-dhdgg 1/1 Running 0 16m -snmp-splunk-connect-for-snmp-worker-trap-6b8fd89868-wcwq5 0/1 Pending 0 16m +NAME READY STATUS RESTARTS AGE +snmp-mibserver-6fdcdf9ddd-7bvmj 1/1 Running 0 25h +snmp-mongodb-6dc5c4f74d-6b7mf 2/2 Running 0 25h +snmp-redis-master-0 1/1 Running 0 25h +snmp-splunk-connect-for-snmp-inventory-sssgs 0/1 Completed 0 3m37s +snmp-splunk-connect-for-snmp-scheduler-5fcb6dcb44-r79ff 1/1 Running 0 25h +snmp-splunk-connect-for-snmp-trap-5788bc498c-62xsq 1/1 Running 0 2m10s +snmp-splunk-connect-for-snmp-trap-5788bc498c-bmlhg 1/1 Running 0 2m10s +snmp-splunk-connect-for-snmp-trap-5788bc498c-p7mkq 1/1 Running 0 2m10s +snmp-splunk-connect-for-snmp-trap-5788bc498c-t8q9c 1/1 Running 0 2m10s +snmp-splunk-connect-for-snmp-trap-5788bc498c-xjjp2 1/1 Running 0 24h +snmp-splunk-connect-for-snmp-worker-poller-5d76b9b675-25tbf 1/1 Running 0 16m +snmp-splunk-connect-for-snmp-worker-poller-5d76b9b675-dc6zr 1/1 Running 0 16m +snmp-splunk-connect-for-snmp-worker-poller-5d76b9b675-g7vpr 1/1 Running 0 16m +snmp-splunk-connect-for-snmp-worker-poller-5d76b9b675-gdkgq 1/1 Running 0 16m +snmp-splunk-connect-for-snmp-worker-poller-5d76b9b675-pg6cj 1/1 Running 0 24h +snmp-splunk-connect-for-snmp-worker-sender-7757fb7f89-56h9w 1/1 Running 0 24h +snmp-splunk-connect-for-snmp-worker-sender-7757fb7f89-hr54w 1/1 Running 0 16m +snmp-splunk-connect-for-snmp-worker-sender-7757fb7f89-j7wcn 1/1 Running 0 16m +snmp-splunk-connect-for-snmp-worker-sender-7757fb7f89-sgsdg 0/1 Pending 0 16m +snmp-splunk-connect-for-snmp-worker-sender-7757fb7f89-xrpfx 1/1 Running 0 16m +snmp-splunk-connect-for-snmp-worker-discovery-7d9fdc5d56-js474 1/1 Running 0 24h +snmp-splunk-connect-for-snmp-worker-discovery-7d9fdc5d56-bfgr4 1/1 Running 0 16m +snmp-splunk-connect-for-snmp-worker-discovery-7d9fdc5d56-gt4rf 1/1 Running 0 16m +snmp-splunk-connect-for-snmp-worker-discovery-7d9fdc5d56-ku76g 0/1 Pending 0 16m +snmp-splunk-connect-for-snmp-worker-discovery-7d9fdc5d56-a243g 1/1 Running 0 16m +snmp-splunk-connect-for-snmp-worker-trap-6b8fd89868-79x2l 0/1 Pending 0 16m +snmp-splunk-connect-for-snmp-worker-trap-6b8fd89868-br7pf 1/1 Running 0 24h +snmp-splunk-connect-for-snmp-worker-trap-6b8fd89868-cnmh9 0/1 Pending 0 16m +snmp-splunk-connect-for-snmp-worker-trap-6b8fd89868-dhdgg 1/1 Running 0 16m +snmp-splunk-connect-for-snmp-worker-trap-6b8fd89868-wcwq5 0/1 Pending 0 16m ``` \ No newline at end of file diff --git a/docs/microk8s/sc4snmp-installation.md b/docs/microk8s/sc4snmp-installation.md index d641a7a4d..761f83be9 100644 --- a/docs/microk8s/sc4snmp-installation.md +++ b/docs/microk8s/sc4snmp-installation.md @@ -94,16 +94,17 @@ microk8s kubectl get pods -n sc4snmp Example output: ``` -NAME READY STATUS RESTARTS AGE -snmp-splunk-connect-for-snmp-scheduler-7ddbc8d75-bljsj 1/1 Running 0 133m -snmp-splunk-connect-for-snmp-worker-poller-57cd8f4665-9z9vx 1/1 Running 0 133m -snmp-splunk-connect-for-snmp-worker-sender-5c44cbb9c5-ppmb5 1/1 Running 0 133m -snmp-splunk-connect-for-snmp-worker-trap-549766d4-28qzh 1/1 Running 0 133m -snmp-mibserver-7f879c5b7c-hz9tz 1/1 Running 0 133m -snmp-mongodb-869cc8586f-vvr9f 2/2 Running 0 133m -snmp-redis-master-0 1/1 Running 0 133m -snmp-splunk-connect-for-snmp-trap-78759bfc8b-79m6d 1/1 Running 0 99m -snmp-splunk-connect-for-snmp-inventory-mjccw 0/1 Completed 0 6s +NAME READY STATUS RESTARTS AGE +snmp-splunk-connect-for-snmp-scheduler-7ddbc8d75-bljsj 1/1 Running 0 133m +snmp-splunk-connect-for-snmp-worker-poller-57cd8f4665-9z9vx 1/1 Running 0 133m +snmp-splunk-connect-for-snmp-worker-sender-5c44cbb9c5-ppmb5 1/1 Running 0 133m +snmp-splunk-connect-for-snmp-worker-trap-549766d4-28qzh 1/1 Running 0 133m +snmp-splunk-connect-for-snmp-worker-discovery-7d9fdc5d56-js474 1/1 Running 0 133m +snmp-mibserver-7f879c5b7c-hz9tz 1/1 Running 0 133m +snmp-mongodb-869cc8586f-vvr9f 2/2 Running 0 133m +snmp-redis-master-0 1/1 Running 0 133m +snmp-splunk-connect-for-snmp-trap-78759bfc8b-79m6d 1/1 Running 0 99m +snmp-splunk-connect-for-snmp-inventory-mjccw 0/1 Completed 0 6s ``` The output might vary depending on the configuration. In the above example, both polling and traps are configured, @@ -214,13 +215,14 @@ To uninstall SC4SNMP run the following commands: Example of pods terminating: ``` -NAME READY STATUS RESTARTS AGE -snmp-mibserver-bb8994c64-twk42 1/1 Terminating 2 (5h21m ago) 46h -snmp-splunk-connect-for-snmp-worker-sender-7f5557678b-psj97 1/1 Terminating 1 (5h21m ago) 22h -snmp-splunk-connect-for-snmp-worker-trap-dfcc487c-lh2dl 1/1 Terminating 1 (5h21m ago) 22h -snmp-splunk-connect-for-snmp-worker-trap-dfcc487c-5z5sq 1/1 Terminating 1 (5h21m ago) 22h -snmp-splunk-connect-for-snmp-trap-684d57dc8d-722tv 1/1 Terminating 1 (5h21m ago) 22h -snmp-splunk-connect-for-snmp-trap-684d57dc8d-z68lb 1/1 Terminating 1 (5h21m ago) 22h +NAME READY STATUS RESTARTS AGE +snmp-mibserver-bb8994c64-twk42 1/1 Terminating 2 (5h21m ago) 46h +snmp-splunk-connect-for-snmp-worker-sender-7f5557678b-psj97 1/1 Terminating 1 (5h21m ago) 22h +snmp-splunk-connect-for-snmp-worker-trap-dfcc487c-lh2dl 1/1 Terminating 1 (5h21m ago) 22h +snmp-splunk-connect-for-snmp-worker-discovery-7d9fdc5d56-js474 1/1 Terminating 1 (5h21m ago) 22h +snmp-splunk-connect-for-snmp-worker-trap-dfcc487c-5z5sq 1/1 Terminating 1 (5h21m ago) 22h +snmp-splunk-connect-for-snmp-trap-684d57dc8d-722tv 1/1 Terminating 1 (5h21m ago) 22h +snmp-splunk-connect-for-snmp-trap-684d57dc8d-z68lb 1/1 Terminating 1 (5h21m ago) 22h ``` ## Restart Splunk Connect for SNMP diff --git a/docs/troubleshooting/discovery-issues.md b/docs/troubleshooting/discovery-issues.md new file mode 100644 index 000000000..e135c3053 --- /dev/null +++ b/docs/troubleshooting/discovery-issues.md @@ -0,0 +1,70 @@ +# Troubleshooting Discovery Issues + + +## Permission denied while writing discovery file + +Discovery fails with a `PermissionError` related to `discovery_devices.csv`. In such cases, you may see the following error: + +```log +PermissionError: [Errno 13] Permission denied: '/app/discovery/discovery_devices.csv' +``` + +The folder specified in the `discoveryPath` value (which is mounted to `/app/discovery` inside the container) does not have the correct permissions for the application user (UID `10001`) to write files. + +Ensure that the folder specified in the `discoveryPath` has write permissions for UID `10001`. You can fix this by updating the folder ownership or permissions before starting the containers. + +**Example (on the host system):** +```bash +sudo chown 10001:10001 /your/local/folder/path +sudo chmod 755 /your/local/folder/path +``` + +## Discovery not completed within the time limit + +If the subnet being scanned has a large IP range (e.g., `/22`, `/21`, or bigger), the task may not be completed within the default time limit of **2400 seconds**. In such cases, you may see the following error: + +```log +[2025-08-07 06:03:29,415: ERROR/MainProcess] Hard time limit (2400s) exceeded for splunk_connect_for_snmp.discovery.tasks.discovery +``` + + +Increase the task timeout value using the `taskTimeout` field under the `worker` section in your `values.yaml`: + +```yaml +worker: + taskTimeout: 3600 # Increase based on expected duration +``` + +## Discovery takes too much time + +Discovery tasks may take longer to complete due to unnecessary SNMP requests or long wait times when scanning large subnets. Below are few ways to optimize performance: + +### Adjust Timeout and Retries + +If the subnet has very few SNMP-enabled devices, high timeout and retry values can significantly slow down the process. +For example, with the default `udpConnectionTimeout` of `3` seconds and `udpConnectionRetries` of `5`, a non-SNMP-enabled device will take up to **15 seconds** before moving to the next IP. +Consider lowering the retry parameters to speed up execution: + +```yaml +worker: + udpConnectionTimeout: 3 + udpConnectionRetries: 2 +``` + +!!! info + Reduce these values carefully. Setting them too low may cause missed detections in slow or high-latency networks, which can impact data accuracy. + +## No Output in `discovery_devices.csv` + +After running a discovery task, no entries are written to the `discovery_devices.csv` file. The issue might have several root causes. Some of them are: + +- Wrong device IP or port provided. +- Subnet contains no reachable or SNMP-enabled devices. +- Nmap is unable to detect live hosts due to network or firewall restrictions. +- For SNMPv2c: Incorrect community string. +- For SNMPv3: Incorrect privacy key or authentication credentials. + +**Resolution:** +- Double-check the IP range or subnet provided in the discovery config. +- Validate that the target devices have SNMP enabled and are reachable from the container. +- Verify SNMP credentials (community string or SNMPv3 credentials) for correctness. diff --git a/entrypoint.sh b/entrypoint.sh index 6ff9e1e04..2d301665f 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -28,6 +28,10 @@ wait-for-dep "${CELERY_BROKER_URL}" "${REDIS_URL}" "${MONGO_URI}" "${MIB_INDEX}" case $1 in +discovery) + discovery-loader + ;; + inventory) inventory-loader ;; @@ -37,6 +41,9 @@ celery) beat) celery -A splunk_connect_for_snmp.poller beat -l "$LOG_LEVEL" --max-interval=10 ;; + worker-discovery) + celery -A splunk_connect_for_snmp.poller worker -l "$LOG_LEVEL" -Q discovery --autoscale=8,"$WORKER_CONCURRENCY" + ;; worker-trap) celery -A splunk_connect_for_snmp.poller worker -l "$LOG_LEVEL" -Q traps --autoscale=8,"$WORKER_CONCURRENCY" ;; diff --git a/examples/polling_and_traps_v3.yaml b/examples/polling_and_traps_v3.yaml index 9eb76e497..dac79f81f 100644 --- a/examples/polling_and_traps_v3.yaml +++ b/examples/polling_and_traps_v3.yaml @@ -7,7 +7,7 @@ splunk: port: "8088" traps: # Remember to create sc4snmp-homesecure-sha-aes and sc4snmp-homesecure-sha-des secrets beforehand - # this is how to do it: https://splunk.github.io/splunk-connect-for-snmp/main/microk8s/microk8s/configuration/snmpv3-configuration/ + # this is how to do it: https://splunk.github.io/splunk-connect-for-snmp/main/microk8s/configuration/snmpv3-configuration/ usernameSecrets: - sc4snmp-homesecure-sha-aes - sc4snmp-homesecure-sha-des diff --git a/integration_tests/.env b/integration_tests/.env index fc2103e08..91337163e 100644 --- a/integration_tests/.env +++ b/integration_tests/.env @@ -5,6 +5,7 @@ SCHEDULER_CONFIG_FILE_ABSOLUTE_PATH= TRAPS_CONFIG_FILE_ABSOLUTE_PATH= INVENTORY_FILE_ABSOLUTE_PATH= COREFILE_ABS_PATH= +DISCOVERY_CONFIG_FILE_ABSOLUTE_PATH= COREDNS_ADDRESS=172.28.0.255 COREDNS_ADDRESS_IPv6=fd02:0:0:0:7fff:ffff:ffff:ffff SC4SNMP_VERSION=latest @@ -35,6 +36,11 @@ SPLUNK_HEC_PATH=/services/collector SPLUNK_AGGREGATE_TRAPS_EVENTS=false IGNORE_EMPTY_VARBINDS=false +# Discovery +DISCOVERY_ENABLE=true +DISCOVERY_LOG_LEVEL=INFO +DISCOVERY_PATH= + # Workers configration WALK_RETRY_MAX_INTERVAL=180 WALK_MAX_RETRIES=5 @@ -75,6 +81,15 @@ WORKER_TRAP_MEMORY_LIMIT=500M WORKER_TRAP_CPU_RESERVATIONS=0.5 WORKER_TRAP_MEMORY_RESERVATIONS=250M +# Worker Discovery +WORKER_DISCOVERY_CONCURRENCY=4 +PREFETCH_DISCOVERY_COUNT=30 +WORKER_DISCOVERY_REPLICAS=1 +WORKER_DISCOVERY_CPU_LIMIT=1 +WORKER_DISCOVERY_MEMORY_LIMIT=500M +WORKER_DISCOVERY_CPU_RESERVATIONS=0.5 +WORKER_DISCOVERY_MEMORY_RESERVATIONS=250M + # Inventory configuration INVENTORY_LOG_LEVEL=INFO CHAIN_OF_TASKS_EXPIRY_TIME=500 diff --git a/integration_tests/automatic_setup_compose.sh b/integration_tests/automatic_setup_compose.sh index 1579fa751..202fb7bce 100755 --- a/integration_tests/automatic_setup_compose.sh +++ b/integration_tests/automatic_setup_compose.sh @@ -104,15 +104,19 @@ cp ../docker_compose/* . SCHEDULER_CONFIG_FILE="scheduler-config.yaml" TRAPS_CONFIG_FILE="traps-config.yaml" INVENTORY_FILE="inventory-tests.csv" +DISCOVERY_CONFIG_FILE="discovery-config.yaml" COREFILE="Corefile" +DISCOVERY_FOLDER="discovery" # Get the absolute paths of the files SCHEDULER_CONFIG_FILE_ABSOLUTE_PATH=$(realpath "$SCHEDULER_CONFIG_FILE") TRAPS_CONFIG_FILE_ABSOLUTE_PATH=$(realpath "$TRAPS_CONFIG_FILE") +DISCOVERY_CONFIG_FILE_ABSOLUTE_PATH=$(realpath "$DISCOVERY_CONFIG_FILE") INVENTORY_FILE_ABSOLUTE_PATH=$(realpath "$INVENTORY_FILE") COREFILE_ABS_PATH=$(realpath "$COREFILE") SPLUNK_HEC_HOST=$(hostname -I | cut -d " " -f1) SPLUNK_HEC_TOKEN=$(cat hec_token) +DISCOVERY_PATH=$(realpath "$DISCOVERY_FOLDER") # Temporary file to store the updated .env content TEMP_ENV_FILE=".env.tmp" @@ -120,16 +124,20 @@ TEMP_ENV_FILE=".env.tmp" # Update or add the variables in the .env file awk -v scheduler_path="$SCHEDULER_CONFIG_FILE_ABSOLUTE_PATH" \ -v traps_path="$TRAPS_CONFIG_FILE_ABSOLUTE_PATH" \ + -v discovery_config_path="$DISCOVERY_CONFIG_FILE_ABSOLUTE_PATH" \ -v inventory_path="$INVENTORY_FILE_ABSOLUTE_PATH" \ -v corefile_path="$COREFILE_ABS_PATH" \ + -v discovery_path="$DISCOVERY_PATH" \ -v splunk_hec_host="$SPLUNK_HEC_HOST" \ -v splunk_hec_token="$SPLUNK_HEC_TOKEN" \ ' BEGIN { updated["SCHEDULER_CONFIG_FILE_ABSOLUTE_PATH"] = 0; updated["TRAPS_CONFIG_FILE_ABSOLUTE_PATH"] = 0; + updated["DICOVERY_CONFIG_FILE_ABSOLUTE_PATH"] = 0; updated["INVENTORY_FILE_ABSOLUTE_PATH"] = 0; updated["COREFILE_ABS_PATH"] = 0; + updated["DISCOVERY_PATH"] = 0; updated["SPLUNK_HEC_HOST"] = 0; updated["SPLUNK_HEC_TOKEN"] = 0; } @@ -140,12 +148,18 @@ awk -v scheduler_path="$SCHEDULER_CONFIG_FILE_ABSOLUTE_PATH" \ } else if ($1 == "TRAPS_CONFIG_FILE_ABSOLUTE_PATH=") { print "TRAPS_CONFIG_FILE_ABSOLUTE_PATH=" traps_path; updated["TRAPS_CONFIG_FILE_ABSOLUTE_PATH"] = 1; + } else if ($1 == "DISCOVERY_CONFIG_FILE_ABSOLUTE_PATH=") { + print "DISCOVERY_CONFIG_FILE_ABSOLUTE_PATH=" discovery_config_path; + updated["DISCOVERY_CONFIG_FILE_ABSOLUTE_PATH"] = 1; } else if ($1 == "INVENTORY_FILE_ABSOLUTE_PATH=") { print "INVENTORY_FILE_ABSOLUTE_PATH=" inventory_path; updated["INVENTORY_FILE_ABSOLUTE_PATH"] = 1; } else if ($1 == "COREFILE_ABS_PATH=") { print "COREFILE_ABS_PATH=" corefile_path; updated["COREFILE_ABS_PATH"] = 1; + } else if ($1 == "DISCOVERY_PATH=") { + print "DISCOVERY_PATH=" discovery_path; + updated["DISCOVERY_PATH"] = 1; } else if ($1 == "SPLUNK_HEC_HOST=") { print "SPLUNK_HEC_HOST=" splunk_hec_host; updated["SPLUNK_HEC_HOST"] = 1; @@ -163,12 +177,18 @@ awk -v scheduler_path="$SCHEDULER_CONFIG_FILE_ABSOLUTE_PATH" \ if (updated["TRAPS_CONFIG_FILE_ABSOLUTE_PATH"] == 0) { print "TRAPS_CONFIG_FILE_ABSOLUTE_PATH=" traps_path; } + if (updated["DISCOVERY_CONFIG_FILE_ABSOLUTE_PATH"] == 0) { + print "DISCOVERY_CONFIG_FILE_ABSOLUTE_PATH=" discovery_config_path; + } if (updated["INVENTORY_FILE_ABSOLUTE_PATH"] == 0) { print "INVENTORY_FILE_ABSOLUTE_PATH=" inventory_path; } if (updated["COREFILE_ABS_PATH"] == 0) { print "COREFILE_ABS_PATH=" corefile_path; } + if (updated["DISCOVERY_PATH"] == 0) { + print "DISCOVERY_PATH=" discovery_path; + } if (updated["SPLUNK_HEC_HOST"] == 0) { print "SPLUNK_HEC_HOST=" splunk_hec_host; } diff --git a/integration_tests/discovery-config.yaml b/integration_tests/discovery-config.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/mkdocs.yml b/mkdocs.yml index eefd07ceb..3e32c4589 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -51,6 +51,7 @@ nav: - Inventory configuration: "dockercompose/3-inventory-configuration.md" - Scheduler configuration: "dockercompose/4-scheduler-configuration.md" - Traps configuration: "dockercompose/5-traps-configuration.md" + - Discovery configuration: "dockercompose/11-discovery-configuration.md" - .env file configuration: "dockercompose/6-env-file-configuration.md" - SNMPv3 secrets configuration: "dockercompose/7-snmpv3-secrets.md" - Offline installation: "dockercompose/8-offline-installation.md" @@ -76,6 +77,7 @@ nav: - SNMP data format: "microk8s/configuration/snmp-data-format.md" - Traps: "microk8s/configuration/trap-configuration.md" - Worker: "microk8s/configuration/worker-configuration.md" + - Discovery: "microk8s/configuration/discovery-configuration.md" - MongoDB: "microk8s/configuration/mongo-configuration.md" - Redis: "microk8s/configuration/redis-configuration.md" - SNMPv3 configuration: "microk8s/configuration/snmpv3-configuration.md" @@ -95,6 +97,7 @@ nav: - High Availability: "ha.md" - Lightweight installation: "small-environment.md" - Splunk dashboards: "dashboard.md" + - Discovery: "discovery.md" - Releases: "releases.md" - Request MIB: "mib-request.md" - Security: "security.md" @@ -106,6 +109,7 @@ nav: - General issues: "troubleshooting/general-issues.md" - Polling issues: "troubleshooting/polling-issues.md" - Traps issues: "troubleshooting/traps-issues.md" + - Discovery issues: "troubleshooting/discovery-issues.md" diff --git a/poetry.lock b/poetry.lock index 3f3dad23f..803e786c4 100644 --- a/poetry.lock +++ b/poetry.lock @@ -518,6 +518,18 @@ typing-extensions = {version = ">=4.6.0", markers = "python_version < \"3.13\""} [package.extras] test = ["pytest (>=6)"] +[[package]] +name = "filelock" +version = "3.19.1" +description = "A platform independent file lock." +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "filelock-3.19.1-py3-none-any.whl", hash = "sha256:d38e30481def20772f5baf097c122c3babc4fcdb7e14e57049eb9d88c6dc017d"}, + {file = "filelock-3.19.1.tar.gz", hash = "sha256:66eda1888b0171c998b35be2bcc0f6d75c388a7ce20c3f3f37aa8e96c2dddf58"}, +] + [[package]] name = "flower" version = "2.0.1" @@ -2600,4 +2612,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.1" python-versions = ">=3.10,<3.12" -content-hash = "1f0b0fd25a8222113dfded6c013d08b5186e14b65080429666d1f90f73a59490" +content-hash = "4f634dbd196f0c6c440b6339dfe8287b6174a41d7615d803027c7494d7285300" diff --git a/pyproject.toml b/pyproject.toml index ae5171ca2..b19923fa9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,6 +10,7 @@ include = ["splunk_connect_for_snmp/profiles/*.yaml"] [tool.poetry.scripts] traps = 'splunk_connect_for_snmp.traps:main' inventory-loader = 'splunk_connect_for_snmp.inventory.loader:load' +discovery-loader = 'splunk_connect_for_snmp.discovery.loader:load' run-walk = 'splunk_connect_for_snmp.walk:run_walk' [tool.pytest.ini_options] @@ -47,6 +48,7 @@ pysnmplib = {git = "https://github.com/pysnmp/pysnmp.git", branch = "main"} urllib3 = "^2.0.0" jsonschema = "4.24.0" flower = "^2.0.1" +filelock = "^3.18.0" [tool.poetry.group.dev.dependencies] pytest = "^8.0.0" diff --git a/rendered/manifests/tests/splunk-connect-for-snmp/templates/common/discovery-config.yaml b/rendered/manifests/tests/splunk-connect-for-snmp/templates/common/discovery-config.yaml new file mode 100644 index 000000000..f19ee31a3 --- /dev/null +++ b/rendered/manifests/tests/splunk-connect-for-snmp/templates/common/discovery-config.yaml @@ -0,0 +1,28 @@ +--- +# Source: splunk-connect-for-snmp/templates/common/discovery-config.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: splunk-connect-for-snmp-discovery-config +data: + discovery-config.yaml: | + autodiscovery: + discovery_version2c: + community: public + delete_already_discovered: false + device_rules: + - group: linux_group + name: Linux servers + patterns: '*linux*' + - group: centos_group + name: Centos servers + patterns: '*centos*' + frequency: 21600 + network_address: 54.82.41.24/28 + port: 161 + version: 2c + discoveryPath: /home/devuser/discovery + enabled: true + ipv6Enabled: true + logLevel: INFO + usernameSecrets: [] diff --git a/rendered/manifests/tests/splunk-connect-for-snmp/templates/discovery/job.yaml b/rendered/manifests/tests/splunk-connect-for-snmp/templates/discovery/job.yaml new file mode 100644 index 000000000..245f36f10 --- /dev/null +++ b/rendered/manifests/tests/splunk-connect-for-snmp/templates/discovery/job.yaml @@ -0,0 +1,64 @@ +--- +# Source: splunk-connect-for-snmp/templates/discovery/job.yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: release-name-splunk-connect-for-snmp-discovery + labels: + app.kubernetes.io/name: splunk-connect-for-snmp-discovery + app.kubernetes.io/instance: release-name + helm.sh/chart: splunk-connect-for-snmp-CURRENT-VERSION + app.kubernetes.io/version: "CURRENT-VERSION" + app.kubernetes.io/managed-by: Helm +spec: + ttlSecondsAfterFinished: 300 + template: + metadata: + # + + labels: + app.kubernetes.io/name: splunk-connect-for-snmp-discovery + app.kubernetes.io/instance: release-name + spec: + containers: + - name: splunk-connect-for-snmp-discovery + image: "ghcr.io/splunk/splunk-connect-for-snmp/container:CURRENT-VERSION" + imagePullPolicy: Always + args: + ["discovery"] + env: + - name: REDIS_HOST + value: release-name-redis + - name: REDIS_PORT + value: "6379" + - name: REDIS_DB + value: "1" + - name: CELERY_DB + value: "0" + - name: DISCOVERY_CONFIG_PATH + value: /app/discovery/discovery-config.yaml + - name: LOG_LEVEL + value: INFO + - name: CHAIN_OF_TASKS_EXPIRY_TIME + value: "60" + - name: CELERY_TASK_TIMEOUT + value: "2400" + volumeMounts: + - name: discovery-config + mountPath: "/app/discovery" + readOnly: true + - name: tmp + mountPath: "/tmp/" + readOnly: false + + volumes: + # # You set volumes at the Pod level, then mount them into containers inside that Pod + - name: discovery-config + configMap: + name: splunk-connect-for-snmp-discovery-config + items: + - key: "discovery-config.yaml" + path: "discovery-config.yaml" + - name: tmp + emptyDir: {} + restartPolicy: OnFailure diff --git a/rendered/manifests/tests/splunk-connect-for-snmp/templates/inventory/job.yaml b/rendered/manifests/tests/splunk-connect-for-snmp/templates/inventory/job.yaml index 3ea664cc0..2708b9ffd 100644 --- a/rendered/manifests/tests/splunk-connect-for-snmp/templates/inventory/job.yaml +++ b/rendered/manifests/tests/splunk-connect-for-snmp/templates/inventory/job.yaml @@ -28,12 +28,16 @@ spec: env: - name: CONFIG_PATH value: /app/config/config.yaml - - name: REDIS_URL - value: redis://release-name-redis-master:6379/1 + - name: REDIS_HOST + value: release-name-redis + - name: REDIS_PORT + value: "6379" + - name: REDIS_DB + value: "1" + - name: CELERY_DB + value: "0" - name: INVENTORY_PATH value: /app/inventory/inventory.csv - - name: CELERY_BROKER_URL - value: redis://release-name-redis-master:6379/0 - name: MONGO_URI value: mongodb://release-name-mongodb:27017 - name: MIB_SOURCES diff --git a/rendered/manifests/tests/splunk-connect-for-snmp/templates/redis/redis-config.yaml b/rendered/manifests/tests/splunk-connect-for-snmp/templates/redis/redis-config.yaml new file mode 100644 index 000000000..22da33840 --- /dev/null +++ b/rendered/manifests/tests/splunk-connect-for-snmp/templates/redis/redis-config.yaml @@ -0,0 +1,33 @@ +--- +# Source: splunk-connect-for-snmp/templates/redis/redis-config.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: release-name-redis-config + namespace: default + labels: + app: release-name-redis +data: + redis.conf: | + # Data directory + dir /data + + # Persistence - RDB + save 900 1 + save 300 10 + save 60 10000 + + # Persistence - AOF + appendonly yes + appendfsync everysec + + # Logging + loglevel notice + + # Memory + maxmemory-policy noeviction + + # Network + bind 0.0.0.0 + protected-mode no + port 6379 diff --git a/rendered/manifests/tests/splunk-connect-for-snmp/templates/redis/redis-standalone-service.yaml b/rendered/manifests/tests/splunk-connect-for-snmp/templates/redis/redis-standalone-service.yaml new file mode 100644 index 000000000..5b3445437 --- /dev/null +++ b/rendered/manifests/tests/splunk-connect-for-snmp/templates/redis/redis-standalone-service.yaml @@ -0,0 +1,15 @@ +--- +# Source: splunk-connect-for-snmp/templates/redis/redis-standalone-service.yaml +apiVersion: v1 +kind: Service +metadata: + name: release-name-redis + namespace: default +spec: + type: ClusterIP + ports: + - port: 6379 + targetPort: 6379 + name: redis + selector: + app: release-name-redis diff --git a/rendered/manifests/tests/splunk-connect-for-snmp/templates/redis/redis-standalone-statefulset.yaml b/rendered/manifests/tests/splunk-connect-for-snmp/templates/redis/redis-standalone-statefulset.yaml new file mode 100644 index 000000000..b38e4ba21 --- /dev/null +++ b/rendered/manifests/tests/splunk-connect-for-snmp/templates/redis/redis-standalone-statefulset.yaml @@ -0,0 +1,112 @@ +--- +# Source: splunk-connect-for-snmp/templates/redis/redis-standalone-statefulset.yaml +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: release-name-redis + namespace: default + labels: + app: release-name-redis +spec: + serviceName: release-name-redis + replicas: 1 + selector: + matchLabels: + app: release-name-redis + template: + metadata: + labels: + app: release-name-redis + annotations: + checksum/config: e82c09fa615350d9c147a0884485f953308babd6b8842d0cbe695ed5595eb530 + spec: + securityContext: + runAsUser: 999 + fsGroup: 999 + initContainers: + - name: fix-permissions + image: redis:8.2.2 + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - | + echo "=== Redis Init: Fixing Permissions ===" + echo "Current ownership:" + ls -ln /data + echo "" + echo "Fixing ownership to 999:999..." + chown -R 999:999 /data + chmod -R 755 /data + echo "" + echo "New ownership:" + ls -ln /data + echo "=== Permissions Fixed ===" + volumeMounts: + - name: redis-data + mountPath: /data + securityContext: + runAsUser: 0 # Must run as root to chown + containers: + - name: redis + image: redis:8.2.2 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 6379 + name: redis + command: + - sh + - -c + args: + - | + # Copy config to writable location + cp /etc/redis/redis.conf /tmp/redis.conf + + # Start Redis + exec redis-server /tmp/redis.conf + volumeMounts: + - name: redis-data + mountPath: /data + - name: redis-config + mountPath: /etc/redis + resources: + limits: + cpu: 500m + memory: 512Mi + requests: + cpu: 250m + memory: 256Mi + livenessProbe: + exec: + command: + - sh + - -c + - | + redis-cli ping + initialDelaySeconds: 30 + periodSeconds: 10 + readinessProbe: + exec: + command: + - sh + - -c + - | + redis-cli ping + initialDelaySeconds: 5 + periodSeconds: 5 + # Storage enabled but no existing PVC - use volumeClaimTemplates below + volumes: + - name: redis-config + configMap: + name: release-name-redis-config + # No existing PVC found, create new one via volumeClaimTemplates + volumeClaimTemplates: + - metadata: + name: redis-data + spec: + accessModes: + - ReadWriteOnce + storageClassName: microk8s-hostpath + resources: + requests: + storage: 5Gi diff --git a/rendered/manifests/tests/splunk-connect-for-snmp/templates/scheduler/deployment.yaml b/rendered/manifests/tests/splunk-connect-for-snmp/templates/scheduler/deployment.yaml index 40c223461..785864c76 100644 --- a/rendered/manifests/tests/splunk-connect-for-snmp/templates/scheduler/deployment.yaml +++ b/rendered/manifests/tests/splunk-connect-for-snmp/templates/scheduler/deployment.yaml @@ -44,10 +44,14 @@ spec: env: - name: CONFIG_PATH value: /app/config/config.yaml - - name: REDIS_URL - value: redis://release-name-redis-master:6379/1 - - name: CELERY_BROKER_URL - value: redis://release-name-redis-master:6379/0 + - name: REDIS_HOST + value: release-name-redis + - name: REDIS_PORT + value: "6379" + - name: REDIS_DB + value: "1" + - name: CELERY_DB + value: "0" - name: MONGO_URI value: mongodb://release-name-mongodb:27017 - name: MIB_SOURCES diff --git a/rendered/manifests/tests/splunk-connect-for-snmp/templates/traps/deployment.yaml b/rendered/manifests/tests/splunk-connect-for-snmp/templates/traps/deployment.yaml index 65c9b834f..db1be6303 100644 --- a/rendered/manifests/tests/splunk-connect-for-snmp/templates/traps/deployment.yaml +++ b/rendered/manifests/tests/splunk-connect-for-snmp/templates/traps/deployment.yaml @@ -44,10 +44,16 @@ spec: env: - name: CONFIG_PATH value: /app/config/config.yaml - - name: CELERY_BROKER_URL - value: redis://release-name-redis-master:6379/0 - name: MONGO_URI value: mongodb://release-name-mongodb:27017 + - name: REDIS_HOST + value: release-name-redis + - name: REDIS_PORT + value: "6379" + - name: REDIS_DB + value: "1" + - name: CELERY_DB + value: "0" - name: MIB_SOURCES value: "http://release-name-mibserver/asn1/@mib@" - name: MIB_INDEX diff --git a/rendered/manifests/tests/splunk-connect-for-snmp/templates/worker/discovery/deployment.yaml b/rendered/manifests/tests/splunk-connect-for-snmp/templates/worker/discovery/deployment.yaml new file mode 100644 index 000000000..121d2a945 --- /dev/null +++ b/rendered/manifests/tests/splunk-connect-for-snmp/templates/worker/discovery/deployment.yaml @@ -0,0 +1,154 @@ +--- +# Source: splunk-connect-for-snmp/templates/worker/discovery/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: release-name-splunk-connect-for-snmp-worker-discovery + labels: + app.kubernetes.io/name: splunk-connect-for-snmp-worker-discovery + app.kubernetes.io/instance: release-name + helm.sh/chart: splunk-connect-for-snmp-CURRENT-VERSION + app.kubernetes.io/version: "CURRENT-VERSION" + app.kubernetes.io/managed-by: Helm +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: splunk-connect-for-snmp-worker-discovery + app.kubernetes.io/instance: release-name + template: + metadata: + labels: + app.kubernetes.io/name: splunk-connect-for-snmp-worker-discovery + app.kubernetes.io/instance: release-name + spec: + serviceAccountName: release-name-splunk-connect-for-snmp-user + securityContext: + fsGroup: 10001 + containers: + - name: splunk-connect-for-snmp-discovery + securityContext: + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 10001 + runAsGroup: 10001 + image: "ghcr.io/splunk/splunk-connect-for-snmp/container:CURRENT-VERSION" + imagePullPolicy: Always + args: + [ + "celery", "worker-discovery", + ] + env: + - name: CONFIG_PATH + value: /app/config/config.yaml + - name: REDIS_HOST + value: release-name-redis + - name: REDIS_PORT + value: "6379" + - name: REDIS_DB + value: "1" + - name: CELERY_DB + value: "0" + - name: SC4SNMP_VERSION + value: CURRENT-VERSION + - name: MONGO_URI + value: mongodb://release-name-mongodb:27017 + - name: WALK_RETRY_MAX_INTERVAL + value: "180" + - name: WALK_MAX_RETRIES + value: "5" + - name: METRICS_INDEXING_ENABLED + value: "false" + - name: POLL_BASE_PROFILES + value: "true" + - name: LOG_LEVEL + value: INFO + - name: DISABLE_MONGO_DEBUG_LOGGING + value: "true" + - name: UDP_CONNECTION_TIMEOUT + value: "3" + - name: UDP_CONNECTION_RETRIES + value: "5" + - name: MAX_OID_TO_PROCESS + value: "70" + - name: MAX_REPETITIONS + value: "10" + - name: PYSNMP_DEBUG + value: "" + - name: PROFILES_RELOAD_DELAY + value: "60" + - name: MIB_SOURCES + value: "http://release-name-mibserver/asn1/@mib@" + - name: MIB_INDEX + value: "http://release-name-mibserver/index.csv" + - name: MIB_STANDARD + value: "http://release-name-mibserver/standard.txt" + - name: SPLUNK_HEC_SCHEME + value: "https" + - name: SPLUNK_HEC_HOST + value: "10.202.18.152" + - name: IGNORE_EMPTY_VARBINDS + value: "false" + - name: SPLUNK_HEC_PORT + value: "8088" + - name: SPLUNK_HEC_INSECURESSL + value: "true" + - name: SPLUNK_AGGREGATE_TRAPS_EVENTS + value: "false" + - name: SPLUNK_METRIC_NAME_HYPHEN_TO_UNDERSCORE + value: "false" + - name: SPLUNK_HEC_TOKEN + valueFrom: + secretKeyRef: + name: splunk-connect-for-snmp-splunk + key: hec_token + - name: SPLUNK_HEC_INDEX_EVENTS + value: netops + - name: SPLUNK_HEC_INDEX_METRICS + value: netmetrics + - name: SPLUNK_SOURCETYPE_TRAPS + value: "sc4snmp:traps" + - name: SPLUNK_SOURCETYPE_POLLING_EVENTS + value: "sc4snmp:event" + - name: SPLUNK_SOURCETYPE_POLLING_METRICS + value: "sc4snmp:metric" + - name: DISCOVERY_FOLDER_PATH + value: /app/discovery + - name: CELERY_TASK_TIMEOUT + value: "2400" + - name: IPv6_ENABLED + value: "true" + + volumeMounts: + - name: config + mountPath: "/app/config" + readOnly: true + - name: pysnmp-cache-volume + mountPath: "/.pysnmp/" + readOnly: false + - name: tmp + mountPath: "/tmp/" + readOnly: false + - name: discovery-volume + mountPath: /app/discovery + volumes: + # You set volumes at the Pod level, then mount them into containers inside that Pod + - name: config + configMap: + # Provide the name of the ConfigMap you want to mount. + name: splunk-connect-for-snmp-config + # An array of keys from the ConfigMap to create as files + items: + - key: "config.yaml" + path: "config.yaml" + - name: pysnmp-cache-volume + emptyDir: {} + - name: tmp + emptyDir: {} + - name: discovery-volume + hostPath: + path: /home/devuser/discovery + type: Directory diff --git a/rendered/manifests/tests/splunk-connect-for-snmp/templates/worker/poller/deployment.yaml b/rendered/manifests/tests/splunk-connect-for-snmp/templates/worker/poller/deployment.yaml index 01dfdcb78..d5f4381ed 100644 --- a/rendered/manifests/tests/splunk-connect-for-snmp/templates/worker/poller/deployment.yaml +++ b/rendered/manifests/tests/splunk-connect-for-snmp/templates/worker/poller/deployment.yaml @@ -44,12 +44,16 @@ spec: env: - name: CONFIG_PATH value: /app/config/config.yaml - - name: REDIS_URL - value: redis://release-name-redis-master:6379/1 + - name: REDIS_HOST + value: release-name-redis + - name: REDIS_PORT + value: "6379" + - name: REDIS_DB + value: "1" + - name: CELERY_DB + value: "0" - name: SC4SNMP_VERSION value: CURRENT-VERSION - - name: CELERY_BROKER_URL - value: redis://release-name-redis-master:6379/0 - name: MONGO_URI value: mongodb://release-name-mongodb:27017 - name: WALK_RETRY_MAX_INTERVAL @@ -66,6 +70,8 @@ spec: value: "true" - name: UDP_CONNECTION_TIMEOUT value: "3" + - name: UDP_CONNECTION_RETRIES + value: "5" - name: MAX_OID_TO_PROCESS value: "70" - name: MAX_REPETITIONS diff --git a/rendered/manifests/tests/splunk-connect-for-snmp/templates/worker/sender/deployment.yaml b/rendered/manifests/tests/splunk-connect-for-snmp/templates/worker/sender/deployment.yaml index fe08ce4ed..7e2ec223b 100644 --- a/rendered/manifests/tests/splunk-connect-for-snmp/templates/worker/sender/deployment.yaml +++ b/rendered/manifests/tests/splunk-connect-for-snmp/templates/worker/sender/deployment.yaml @@ -44,12 +44,16 @@ spec: env: - name: CONFIG_PATH value: /app/config/config.yaml - - name: REDIS_URL - value: redis://release-name-redis-master:6379/1 + - name: REDIS_HOST + value: release-name-redis + - name: REDIS_PORT + value: "6379" + - name: REDIS_DB + value: "1" + - name: CELERY_DB + value: "0" - name: SC4SNMP_VERSION value: CURRENT-VERSION - - name: CELERY_BROKER_URL - value: redis://release-name-redis-master:6379/0 - name: MONGO_URI value: mongodb://release-name-mongodb:27017 - name: WALK_RETRY_MAX_INTERVAL @@ -66,6 +70,8 @@ spec: value: "true" - name: UDP_CONNECTION_TIMEOUT value: "3" + - name: UDP_CONNECTION_RETRIES + value: "5" - name: MAX_OID_TO_PROCESS value: "70" - name: MAX_REPETITIONS diff --git a/rendered/manifests/tests/splunk-connect-for-snmp/templates/worker/trap/deployment.yaml b/rendered/manifests/tests/splunk-connect-for-snmp/templates/worker/trap/deployment.yaml index 82675bad1..d506f7a42 100644 --- a/rendered/manifests/tests/splunk-connect-for-snmp/templates/worker/trap/deployment.yaml +++ b/rendered/manifests/tests/splunk-connect-for-snmp/templates/worker/trap/deployment.yaml @@ -44,12 +44,16 @@ spec: env: - name: CONFIG_PATH value: /app/config/config.yaml - - name: REDIS_URL - value: redis://release-name-redis-master:6379/1 + - name: REDIS_HOST + value: release-name-redis + - name: REDIS_PORT + value: "6379" + - name: REDIS_DB + value: "1" + - name: CELERY_DB + value: "0" - name: SC4SNMP_VERSION value: CURRENT-VERSION - - name: CELERY_BROKER_URL - value: redis://release-name-redis-master:6379/0 - name: MONGO_URI value: mongodb://release-name-mongodb:27017 - name: WALK_RETRY_MAX_INTERVAL @@ -66,6 +70,8 @@ spec: value: "true" - name: UDP_CONNECTION_TIMEOUT value: "3" + - name: UDP_CONNECTION_RETRIES + value: "5" - name: MAX_OID_TO_PROCESS value: "70" - name: MAX_REPETITIONS diff --git a/rendered/manifests/tests_autoscaling_enabled/splunk-connect-for-snmp/templates/common/discovery-config.yaml b/rendered/manifests/tests_autoscaling_enabled/splunk-connect-for-snmp/templates/common/discovery-config.yaml new file mode 100644 index 000000000..8f5acde96 --- /dev/null +++ b/rendered/manifests/tests_autoscaling_enabled/splunk-connect-for-snmp/templates/common/discovery-config.yaml @@ -0,0 +1,12 @@ +--- +# Source: splunk-connect-for-snmp/templates/common/discovery-config.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: splunk-connect-for-snmp-discovery-config +data: + discovery-config.yaml: | + enabled: false + ipv6Enabled: false + logLevel: INFO + usernameSecrets: [] diff --git a/rendered/manifests/tests_autoscaling_enabled/splunk-connect-for-snmp/templates/inventory/job.yaml b/rendered/manifests/tests_autoscaling_enabled/splunk-connect-for-snmp/templates/inventory/job.yaml index 3ea664cc0..2708b9ffd 100644 --- a/rendered/manifests/tests_autoscaling_enabled/splunk-connect-for-snmp/templates/inventory/job.yaml +++ b/rendered/manifests/tests_autoscaling_enabled/splunk-connect-for-snmp/templates/inventory/job.yaml @@ -28,12 +28,16 @@ spec: env: - name: CONFIG_PATH value: /app/config/config.yaml - - name: REDIS_URL - value: redis://release-name-redis-master:6379/1 + - name: REDIS_HOST + value: release-name-redis + - name: REDIS_PORT + value: "6379" + - name: REDIS_DB + value: "1" + - name: CELERY_DB + value: "0" - name: INVENTORY_PATH value: /app/inventory/inventory.csv - - name: CELERY_BROKER_URL - value: redis://release-name-redis-master:6379/0 - name: MONGO_URI value: mongodb://release-name-mongodb:27017 - name: MIB_SOURCES diff --git a/rendered/manifests/tests_autoscaling_enabled/splunk-connect-for-snmp/templates/redis/redis-config.yaml b/rendered/manifests/tests_autoscaling_enabled/splunk-connect-for-snmp/templates/redis/redis-config.yaml new file mode 100644 index 000000000..22da33840 --- /dev/null +++ b/rendered/manifests/tests_autoscaling_enabled/splunk-connect-for-snmp/templates/redis/redis-config.yaml @@ -0,0 +1,33 @@ +--- +# Source: splunk-connect-for-snmp/templates/redis/redis-config.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: release-name-redis-config + namespace: default + labels: + app: release-name-redis +data: + redis.conf: | + # Data directory + dir /data + + # Persistence - RDB + save 900 1 + save 300 10 + save 60 10000 + + # Persistence - AOF + appendonly yes + appendfsync everysec + + # Logging + loglevel notice + + # Memory + maxmemory-policy noeviction + + # Network + bind 0.0.0.0 + protected-mode no + port 6379 diff --git a/rendered/manifests/tests_autoscaling_enabled/splunk-connect-for-snmp/templates/redis/redis-standalone-service.yaml b/rendered/manifests/tests_autoscaling_enabled/splunk-connect-for-snmp/templates/redis/redis-standalone-service.yaml new file mode 100644 index 000000000..5b3445437 --- /dev/null +++ b/rendered/manifests/tests_autoscaling_enabled/splunk-connect-for-snmp/templates/redis/redis-standalone-service.yaml @@ -0,0 +1,15 @@ +--- +# Source: splunk-connect-for-snmp/templates/redis/redis-standalone-service.yaml +apiVersion: v1 +kind: Service +metadata: + name: release-name-redis + namespace: default +spec: + type: ClusterIP + ports: + - port: 6379 + targetPort: 6379 + name: redis + selector: + app: release-name-redis diff --git a/rendered/manifests/tests_autoscaling_enabled/splunk-connect-for-snmp/templates/redis/redis-standalone-statefulset.yaml b/rendered/manifests/tests_autoscaling_enabled/splunk-connect-for-snmp/templates/redis/redis-standalone-statefulset.yaml new file mode 100644 index 000000000..b38e4ba21 --- /dev/null +++ b/rendered/manifests/tests_autoscaling_enabled/splunk-connect-for-snmp/templates/redis/redis-standalone-statefulset.yaml @@ -0,0 +1,112 @@ +--- +# Source: splunk-connect-for-snmp/templates/redis/redis-standalone-statefulset.yaml +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: release-name-redis + namespace: default + labels: + app: release-name-redis +spec: + serviceName: release-name-redis + replicas: 1 + selector: + matchLabels: + app: release-name-redis + template: + metadata: + labels: + app: release-name-redis + annotations: + checksum/config: e82c09fa615350d9c147a0884485f953308babd6b8842d0cbe695ed5595eb530 + spec: + securityContext: + runAsUser: 999 + fsGroup: 999 + initContainers: + - name: fix-permissions + image: redis:8.2.2 + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - | + echo "=== Redis Init: Fixing Permissions ===" + echo "Current ownership:" + ls -ln /data + echo "" + echo "Fixing ownership to 999:999..." + chown -R 999:999 /data + chmod -R 755 /data + echo "" + echo "New ownership:" + ls -ln /data + echo "=== Permissions Fixed ===" + volumeMounts: + - name: redis-data + mountPath: /data + securityContext: + runAsUser: 0 # Must run as root to chown + containers: + - name: redis + image: redis:8.2.2 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 6379 + name: redis + command: + - sh + - -c + args: + - | + # Copy config to writable location + cp /etc/redis/redis.conf /tmp/redis.conf + + # Start Redis + exec redis-server /tmp/redis.conf + volumeMounts: + - name: redis-data + mountPath: /data + - name: redis-config + mountPath: /etc/redis + resources: + limits: + cpu: 500m + memory: 512Mi + requests: + cpu: 250m + memory: 256Mi + livenessProbe: + exec: + command: + - sh + - -c + - | + redis-cli ping + initialDelaySeconds: 30 + periodSeconds: 10 + readinessProbe: + exec: + command: + - sh + - -c + - | + redis-cli ping + initialDelaySeconds: 5 + periodSeconds: 5 + # Storage enabled but no existing PVC - use volumeClaimTemplates below + volumes: + - name: redis-config + configMap: + name: release-name-redis-config + # No existing PVC found, create new one via volumeClaimTemplates + volumeClaimTemplates: + - metadata: + name: redis-data + spec: + accessModes: + - ReadWriteOnce + storageClassName: microk8s-hostpath + resources: + requests: + storage: 5Gi diff --git a/rendered/manifests/tests_autoscaling_enabled/splunk-connect-for-snmp/templates/scheduler/deployment.yaml b/rendered/manifests/tests_autoscaling_enabled/splunk-connect-for-snmp/templates/scheduler/deployment.yaml index 40c223461..785864c76 100644 --- a/rendered/manifests/tests_autoscaling_enabled/splunk-connect-for-snmp/templates/scheduler/deployment.yaml +++ b/rendered/manifests/tests_autoscaling_enabled/splunk-connect-for-snmp/templates/scheduler/deployment.yaml @@ -44,10 +44,14 @@ spec: env: - name: CONFIG_PATH value: /app/config/config.yaml - - name: REDIS_URL - value: redis://release-name-redis-master:6379/1 - - name: CELERY_BROKER_URL - value: redis://release-name-redis-master:6379/0 + - name: REDIS_HOST + value: release-name-redis + - name: REDIS_PORT + value: "6379" + - name: REDIS_DB + value: "1" + - name: CELERY_DB + value: "0" - name: MONGO_URI value: mongodb://release-name-mongodb:27017 - name: MIB_SOURCES diff --git a/rendered/manifests/tests_autoscaling_enabled/splunk-connect-for-snmp/templates/traps/deployment.yaml b/rendered/manifests/tests_autoscaling_enabled/splunk-connect-for-snmp/templates/traps/deployment.yaml index e66ea7f05..552b797a8 100644 --- a/rendered/manifests/tests_autoscaling_enabled/splunk-connect-for-snmp/templates/traps/deployment.yaml +++ b/rendered/manifests/tests_autoscaling_enabled/splunk-connect-for-snmp/templates/traps/deployment.yaml @@ -43,10 +43,16 @@ spec: env: - name: CONFIG_PATH value: /app/config/config.yaml - - name: CELERY_BROKER_URL - value: redis://release-name-redis-master:6379/0 - name: MONGO_URI value: mongodb://release-name-mongodb:27017 + - name: REDIS_HOST + value: release-name-redis + - name: REDIS_PORT + value: "6379" + - name: REDIS_DB + value: "1" + - name: CELERY_DB + value: "0" - name: MIB_SOURCES value: "http://release-name-mibserver/asn1/@mib@" - name: MIB_INDEX diff --git a/rendered/manifests/tests_autoscaling_enabled/splunk-connect-for-snmp/templates/worker/poller/deployment.yaml b/rendered/manifests/tests_autoscaling_enabled/splunk-connect-for-snmp/templates/worker/poller/deployment.yaml index 8a1867d9f..1c8a48f4b 100644 --- a/rendered/manifests/tests_autoscaling_enabled/splunk-connect-for-snmp/templates/worker/poller/deployment.yaml +++ b/rendered/manifests/tests_autoscaling_enabled/splunk-connect-for-snmp/templates/worker/poller/deployment.yaml @@ -43,12 +43,16 @@ spec: env: - name: CONFIG_PATH value: /app/config/config.yaml - - name: REDIS_URL - value: redis://release-name-redis-master:6379/1 + - name: REDIS_HOST + value: release-name-redis + - name: REDIS_PORT + value: "6379" + - name: REDIS_DB + value: "1" + - name: CELERY_DB + value: "0" - name: SC4SNMP_VERSION value: CURRENT-VERSION - - name: CELERY_BROKER_URL - value: redis://release-name-redis-master:6379/0 - name: MONGO_URI value: mongodb://release-name-mongodb:27017 - name: WALK_RETRY_MAX_INTERVAL @@ -65,6 +69,8 @@ spec: value: "true" - name: UDP_CONNECTION_TIMEOUT value: "3" + - name: UDP_CONNECTION_RETRIES + value: "5" - name: MAX_OID_TO_PROCESS value: "70" - name: MAX_REPETITIONS diff --git a/rendered/manifests/tests_autoscaling_enabled/splunk-connect-for-snmp/templates/worker/sender/deployment.yaml b/rendered/manifests/tests_autoscaling_enabled/splunk-connect-for-snmp/templates/worker/sender/deployment.yaml index f89d4ca52..6def28cd1 100644 --- a/rendered/manifests/tests_autoscaling_enabled/splunk-connect-for-snmp/templates/worker/sender/deployment.yaml +++ b/rendered/manifests/tests_autoscaling_enabled/splunk-connect-for-snmp/templates/worker/sender/deployment.yaml @@ -43,12 +43,16 @@ spec: env: - name: CONFIG_PATH value: /app/config/config.yaml - - name: REDIS_URL - value: redis://release-name-redis-master:6379/1 + - name: REDIS_HOST + value: release-name-redis + - name: REDIS_PORT + value: "6379" + - name: REDIS_DB + value: "1" + - name: CELERY_DB + value: "0" - name: SC4SNMP_VERSION value: CURRENT-VERSION - - name: CELERY_BROKER_URL - value: redis://release-name-redis-master:6379/0 - name: MONGO_URI value: mongodb://release-name-mongodb:27017 - name: WALK_RETRY_MAX_INTERVAL @@ -65,6 +69,8 @@ spec: value: "true" - name: UDP_CONNECTION_TIMEOUT value: "3" + - name: UDP_CONNECTION_RETRIES + value: "5" - name: MAX_OID_TO_PROCESS value: "70" - name: MAX_REPETITIONS diff --git a/rendered/manifests/tests_autoscaling_enabled/splunk-connect-for-snmp/templates/worker/trap/deployment.yaml b/rendered/manifests/tests_autoscaling_enabled/splunk-connect-for-snmp/templates/worker/trap/deployment.yaml index 4fbae94b8..c08572b10 100644 --- a/rendered/manifests/tests_autoscaling_enabled/splunk-connect-for-snmp/templates/worker/trap/deployment.yaml +++ b/rendered/manifests/tests_autoscaling_enabled/splunk-connect-for-snmp/templates/worker/trap/deployment.yaml @@ -43,12 +43,16 @@ spec: env: - name: CONFIG_PATH value: /app/config/config.yaml - - name: REDIS_URL - value: redis://release-name-redis-master:6379/1 + - name: REDIS_HOST + value: release-name-redis + - name: REDIS_PORT + value: "6379" + - name: REDIS_DB + value: "1" + - name: CELERY_DB + value: "0" - name: SC4SNMP_VERSION value: CURRENT-VERSION - - name: CELERY_BROKER_URL - value: redis://release-name-redis-master:6379/0 - name: MONGO_URI value: mongodb://release-name-mongodb:27017 - name: WALK_RETRY_MAX_INTERVAL @@ -65,6 +69,8 @@ spec: value: "true" - name: UDP_CONNECTION_TIMEOUT value: "3" + - name: UDP_CONNECTION_RETRIES + value: "5" - name: MAX_OID_TO_PROCESS value: "70" - name: MAX_REPETITIONS diff --git a/rendered/manifests/tests_autoscaling_enabled_deprecated/splunk-connect-for-snmp/templates/common/discovery-config.yaml b/rendered/manifests/tests_autoscaling_enabled_deprecated/splunk-connect-for-snmp/templates/common/discovery-config.yaml new file mode 100644 index 000000000..8f5acde96 --- /dev/null +++ b/rendered/manifests/tests_autoscaling_enabled_deprecated/splunk-connect-for-snmp/templates/common/discovery-config.yaml @@ -0,0 +1,12 @@ +--- +# Source: splunk-connect-for-snmp/templates/common/discovery-config.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: splunk-connect-for-snmp-discovery-config +data: + discovery-config.yaml: | + enabled: false + ipv6Enabled: false + logLevel: INFO + usernameSecrets: [] diff --git a/rendered/manifests/tests_autoscaling_enabled_deprecated/splunk-connect-for-snmp/templates/inventory/job.yaml b/rendered/manifests/tests_autoscaling_enabled_deprecated/splunk-connect-for-snmp/templates/inventory/job.yaml index 3ea664cc0..2708b9ffd 100644 --- a/rendered/manifests/tests_autoscaling_enabled_deprecated/splunk-connect-for-snmp/templates/inventory/job.yaml +++ b/rendered/manifests/tests_autoscaling_enabled_deprecated/splunk-connect-for-snmp/templates/inventory/job.yaml @@ -28,12 +28,16 @@ spec: env: - name: CONFIG_PATH value: /app/config/config.yaml - - name: REDIS_URL - value: redis://release-name-redis-master:6379/1 + - name: REDIS_HOST + value: release-name-redis + - name: REDIS_PORT + value: "6379" + - name: REDIS_DB + value: "1" + - name: CELERY_DB + value: "0" - name: INVENTORY_PATH value: /app/inventory/inventory.csv - - name: CELERY_BROKER_URL - value: redis://release-name-redis-master:6379/0 - name: MONGO_URI value: mongodb://release-name-mongodb:27017 - name: MIB_SOURCES diff --git a/rendered/manifests/tests_autoscaling_enabled_deprecated/splunk-connect-for-snmp/templates/redis/redis-config.yaml b/rendered/manifests/tests_autoscaling_enabled_deprecated/splunk-connect-for-snmp/templates/redis/redis-config.yaml new file mode 100644 index 000000000..22da33840 --- /dev/null +++ b/rendered/manifests/tests_autoscaling_enabled_deprecated/splunk-connect-for-snmp/templates/redis/redis-config.yaml @@ -0,0 +1,33 @@ +--- +# Source: splunk-connect-for-snmp/templates/redis/redis-config.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: release-name-redis-config + namespace: default + labels: + app: release-name-redis +data: + redis.conf: | + # Data directory + dir /data + + # Persistence - RDB + save 900 1 + save 300 10 + save 60 10000 + + # Persistence - AOF + appendonly yes + appendfsync everysec + + # Logging + loglevel notice + + # Memory + maxmemory-policy noeviction + + # Network + bind 0.0.0.0 + protected-mode no + port 6379 diff --git a/rendered/manifests/tests_autoscaling_enabled_deprecated/splunk-connect-for-snmp/templates/redis/redis-standalone-service.yaml b/rendered/manifests/tests_autoscaling_enabled_deprecated/splunk-connect-for-snmp/templates/redis/redis-standalone-service.yaml new file mode 100644 index 000000000..5b3445437 --- /dev/null +++ b/rendered/manifests/tests_autoscaling_enabled_deprecated/splunk-connect-for-snmp/templates/redis/redis-standalone-service.yaml @@ -0,0 +1,15 @@ +--- +# Source: splunk-connect-for-snmp/templates/redis/redis-standalone-service.yaml +apiVersion: v1 +kind: Service +metadata: + name: release-name-redis + namespace: default +spec: + type: ClusterIP + ports: + - port: 6379 + targetPort: 6379 + name: redis + selector: + app: release-name-redis diff --git a/rendered/manifests/tests_autoscaling_enabled_deprecated/splunk-connect-for-snmp/templates/redis/redis-standalone-statefulset.yaml b/rendered/manifests/tests_autoscaling_enabled_deprecated/splunk-connect-for-snmp/templates/redis/redis-standalone-statefulset.yaml new file mode 100644 index 000000000..b38e4ba21 --- /dev/null +++ b/rendered/manifests/tests_autoscaling_enabled_deprecated/splunk-connect-for-snmp/templates/redis/redis-standalone-statefulset.yaml @@ -0,0 +1,112 @@ +--- +# Source: splunk-connect-for-snmp/templates/redis/redis-standalone-statefulset.yaml +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: release-name-redis + namespace: default + labels: + app: release-name-redis +spec: + serviceName: release-name-redis + replicas: 1 + selector: + matchLabels: + app: release-name-redis + template: + metadata: + labels: + app: release-name-redis + annotations: + checksum/config: e82c09fa615350d9c147a0884485f953308babd6b8842d0cbe695ed5595eb530 + spec: + securityContext: + runAsUser: 999 + fsGroup: 999 + initContainers: + - name: fix-permissions + image: redis:8.2.2 + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - | + echo "=== Redis Init: Fixing Permissions ===" + echo "Current ownership:" + ls -ln /data + echo "" + echo "Fixing ownership to 999:999..." + chown -R 999:999 /data + chmod -R 755 /data + echo "" + echo "New ownership:" + ls -ln /data + echo "=== Permissions Fixed ===" + volumeMounts: + - name: redis-data + mountPath: /data + securityContext: + runAsUser: 0 # Must run as root to chown + containers: + - name: redis + image: redis:8.2.2 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 6379 + name: redis + command: + - sh + - -c + args: + - | + # Copy config to writable location + cp /etc/redis/redis.conf /tmp/redis.conf + + # Start Redis + exec redis-server /tmp/redis.conf + volumeMounts: + - name: redis-data + mountPath: /data + - name: redis-config + mountPath: /etc/redis + resources: + limits: + cpu: 500m + memory: 512Mi + requests: + cpu: 250m + memory: 256Mi + livenessProbe: + exec: + command: + - sh + - -c + - | + redis-cli ping + initialDelaySeconds: 30 + periodSeconds: 10 + readinessProbe: + exec: + command: + - sh + - -c + - | + redis-cli ping + initialDelaySeconds: 5 + periodSeconds: 5 + # Storage enabled but no existing PVC - use volumeClaimTemplates below + volumes: + - name: redis-config + configMap: + name: release-name-redis-config + # No existing PVC found, create new one via volumeClaimTemplates + volumeClaimTemplates: + - metadata: + name: redis-data + spec: + accessModes: + - ReadWriteOnce + storageClassName: microk8s-hostpath + resources: + requests: + storage: 5Gi diff --git a/rendered/manifests/tests_autoscaling_enabled_deprecated/splunk-connect-for-snmp/templates/scheduler/deployment.yaml b/rendered/manifests/tests_autoscaling_enabled_deprecated/splunk-connect-for-snmp/templates/scheduler/deployment.yaml index 40c223461..785864c76 100644 --- a/rendered/manifests/tests_autoscaling_enabled_deprecated/splunk-connect-for-snmp/templates/scheduler/deployment.yaml +++ b/rendered/manifests/tests_autoscaling_enabled_deprecated/splunk-connect-for-snmp/templates/scheduler/deployment.yaml @@ -44,10 +44,14 @@ spec: env: - name: CONFIG_PATH value: /app/config/config.yaml - - name: REDIS_URL - value: redis://release-name-redis-master:6379/1 - - name: CELERY_BROKER_URL - value: redis://release-name-redis-master:6379/0 + - name: REDIS_HOST + value: release-name-redis + - name: REDIS_PORT + value: "6379" + - name: REDIS_DB + value: "1" + - name: CELERY_DB + value: "0" - name: MONGO_URI value: mongodb://release-name-mongodb:27017 - name: MIB_SOURCES diff --git a/rendered/manifests/tests_autoscaling_enabled_deprecated/splunk-connect-for-snmp/templates/traps/deployment.yaml b/rendered/manifests/tests_autoscaling_enabled_deprecated/splunk-connect-for-snmp/templates/traps/deployment.yaml index e66ea7f05..552b797a8 100644 --- a/rendered/manifests/tests_autoscaling_enabled_deprecated/splunk-connect-for-snmp/templates/traps/deployment.yaml +++ b/rendered/manifests/tests_autoscaling_enabled_deprecated/splunk-connect-for-snmp/templates/traps/deployment.yaml @@ -43,10 +43,16 @@ spec: env: - name: CONFIG_PATH value: /app/config/config.yaml - - name: CELERY_BROKER_URL - value: redis://release-name-redis-master:6379/0 - name: MONGO_URI value: mongodb://release-name-mongodb:27017 + - name: REDIS_HOST + value: release-name-redis + - name: REDIS_PORT + value: "6379" + - name: REDIS_DB + value: "1" + - name: CELERY_DB + value: "0" - name: MIB_SOURCES value: "http://release-name-mibserver/asn1/@mib@" - name: MIB_INDEX diff --git a/rendered/manifests/tests_autoscaling_enabled_deprecated/splunk-connect-for-snmp/templates/worker/poller/deployment.yaml b/rendered/manifests/tests_autoscaling_enabled_deprecated/splunk-connect-for-snmp/templates/worker/poller/deployment.yaml index 8a1867d9f..1c8a48f4b 100644 --- a/rendered/manifests/tests_autoscaling_enabled_deprecated/splunk-connect-for-snmp/templates/worker/poller/deployment.yaml +++ b/rendered/manifests/tests_autoscaling_enabled_deprecated/splunk-connect-for-snmp/templates/worker/poller/deployment.yaml @@ -43,12 +43,16 @@ spec: env: - name: CONFIG_PATH value: /app/config/config.yaml - - name: REDIS_URL - value: redis://release-name-redis-master:6379/1 + - name: REDIS_HOST + value: release-name-redis + - name: REDIS_PORT + value: "6379" + - name: REDIS_DB + value: "1" + - name: CELERY_DB + value: "0" - name: SC4SNMP_VERSION value: CURRENT-VERSION - - name: CELERY_BROKER_URL - value: redis://release-name-redis-master:6379/0 - name: MONGO_URI value: mongodb://release-name-mongodb:27017 - name: WALK_RETRY_MAX_INTERVAL @@ -65,6 +69,8 @@ spec: value: "true" - name: UDP_CONNECTION_TIMEOUT value: "3" + - name: UDP_CONNECTION_RETRIES + value: "5" - name: MAX_OID_TO_PROCESS value: "70" - name: MAX_REPETITIONS diff --git a/rendered/manifests/tests_autoscaling_enabled_deprecated/splunk-connect-for-snmp/templates/worker/sender/deployment.yaml b/rendered/manifests/tests_autoscaling_enabled_deprecated/splunk-connect-for-snmp/templates/worker/sender/deployment.yaml index f89d4ca52..6def28cd1 100644 --- a/rendered/manifests/tests_autoscaling_enabled_deprecated/splunk-connect-for-snmp/templates/worker/sender/deployment.yaml +++ b/rendered/manifests/tests_autoscaling_enabled_deprecated/splunk-connect-for-snmp/templates/worker/sender/deployment.yaml @@ -43,12 +43,16 @@ spec: env: - name: CONFIG_PATH value: /app/config/config.yaml - - name: REDIS_URL - value: redis://release-name-redis-master:6379/1 + - name: REDIS_HOST + value: release-name-redis + - name: REDIS_PORT + value: "6379" + - name: REDIS_DB + value: "1" + - name: CELERY_DB + value: "0" - name: SC4SNMP_VERSION value: CURRENT-VERSION - - name: CELERY_BROKER_URL - value: redis://release-name-redis-master:6379/0 - name: MONGO_URI value: mongodb://release-name-mongodb:27017 - name: WALK_RETRY_MAX_INTERVAL @@ -65,6 +69,8 @@ spec: value: "true" - name: UDP_CONNECTION_TIMEOUT value: "3" + - name: UDP_CONNECTION_RETRIES + value: "5" - name: MAX_OID_TO_PROCESS value: "70" - name: MAX_REPETITIONS diff --git a/rendered/manifests/tests_autoscaling_enabled_deprecated/splunk-connect-for-snmp/templates/worker/trap/deployment.yaml b/rendered/manifests/tests_autoscaling_enabled_deprecated/splunk-connect-for-snmp/templates/worker/trap/deployment.yaml index 4fbae94b8..c08572b10 100644 --- a/rendered/manifests/tests_autoscaling_enabled_deprecated/splunk-connect-for-snmp/templates/worker/trap/deployment.yaml +++ b/rendered/manifests/tests_autoscaling_enabled_deprecated/splunk-connect-for-snmp/templates/worker/trap/deployment.yaml @@ -43,12 +43,16 @@ spec: env: - name: CONFIG_PATH value: /app/config/config.yaml - - name: REDIS_URL - value: redis://release-name-redis-master:6379/1 + - name: REDIS_HOST + value: release-name-redis + - name: REDIS_PORT + value: "6379" + - name: REDIS_DB + value: "1" + - name: CELERY_DB + value: "0" - name: SC4SNMP_VERSION value: CURRENT-VERSION - - name: CELERY_BROKER_URL - value: redis://release-name-redis-master:6379/0 - name: MONGO_URI value: mongodb://release-name-mongodb:27017 - name: WALK_RETRY_MAX_INTERVAL @@ -65,6 +69,8 @@ spec: value: "true" - name: UDP_CONNECTION_TIMEOUT value: "3" + - name: UDP_CONNECTION_RETRIES + value: "5" - name: MAX_OID_TO_PROCESS value: "70" - name: MAX_REPETITIONS diff --git a/rendered/manifests/tests_enable_ui/splunk-connect-for-snmp/templates/common/discovery-config.yaml b/rendered/manifests/tests_enable_ui/splunk-connect-for-snmp/templates/common/discovery-config.yaml new file mode 100644 index 000000000..8f5acde96 --- /dev/null +++ b/rendered/manifests/tests_enable_ui/splunk-connect-for-snmp/templates/common/discovery-config.yaml @@ -0,0 +1,12 @@ +--- +# Source: splunk-connect-for-snmp/templates/common/discovery-config.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: splunk-connect-for-snmp-discovery-config +data: + discovery-config.yaml: | + enabled: false + ipv6Enabled: false + logLevel: INFO + usernameSecrets: [] diff --git a/rendered/manifests/tests_enable_ui/splunk-connect-for-snmp/templates/inventory/job.yaml b/rendered/manifests/tests_enable_ui/splunk-connect-for-snmp/templates/inventory/job.yaml index 31b6ec595..2acdc1c06 100644 --- a/rendered/manifests/tests_enable_ui/splunk-connect-for-snmp/templates/inventory/job.yaml +++ b/rendered/manifests/tests_enable_ui/splunk-connect-for-snmp/templates/inventory/job.yaml @@ -28,12 +28,16 @@ spec: env: - name: CONFIG_PATH value: /app/config/config.yaml - - name: REDIS_URL - value: redis://release-name-redis-master:6379/1 + - name: REDIS_HOST + value: release-name-redis + - name: REDIS_PORT + value: "6379" + - name: REDIS_DB + value: "1" + - name: CELERY_DB + value: "0" - name: INVENTORY_PATH value: /app/inventory/inventory.csv - - name: CELERY_BROKER_URL - value: redis://release-name-redis-master:6379/0 - name: MONGO_URI value: mongodb://release-name-mongodb:27017 - name: MIB_SOURCES diff --git a/rendered/manifests/tests_enable_ui/splunk-connect-for-snmp/templates/redis/redis-config.yaml b/rendered/manifests/tests_enable_ui/splunk-connect-for-snmp/templates/redis/redis-config.yaml new file mode 100644 index 000000000..22da33840 --- /dev/null +++ b/rendered/manifests/tests_enable_ui/splunk-connect-for-snmp/templates/redis/redis-config.yaml @@ -0,0 +1,33 @@ +--- +# Source: splunk-connect-for-snmp/templates/redis/redis-config.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: release-name-redis-config + namespace: default + labels: + app: release-name-redis +data: + redis.conf: | + # Data directory + dir /data + + # Persistence - RDB + save 900 1 + save 300 10 + save 60 10000 + + # Persistence - AOF + appendonly yes + appendfsync everysec + + # Logging + loglevel notice + + # Memory + maxmemory-policy noeviction + + # Network + bind 0.0.0.0 + protected-mode no + port 6379 diff --git a/rendered/manifests/tests_enable_ui/splunk-connect-for-snmp/templates/redis/redis-standalone-service.yaml b/rendered/manifests/tests_enable_ui/splunk-connect-for-snmp/templates/redis/redis-standalone-service.yaml new file mode 100644 index 000000000..5b3445437 --- /dev/null +++ b/rendered/manifests/tests_enable_ui/splunk-connect-for-snmp/templates/redis/redis-standalone-service.yaml @@ -0,0 +1,15 @@ +--- +# Source: splunk-connect-for-snmp/templates/redis/redis-standalone-service.yaml +apiVersion: v1 +kind: Service +metadata: + name: release-name-redis + namespace: default +spec: + type: ClusterIP + ports: + - port: 6379 + targetPort: 6379 + name: redis + selector: + app: release-name-redis diff --git a/rendered/manifests/tests_enable_ui/splunk-connect-for-snmp/templates/redis/redis-standalone-statefulset.yaml b/rendered/manifests/tests_enable_ui/splunk-connect-for-snmp/templates/redis/redis-standalone-statefulset.yaml new file mode 100644 index 000000000..b38e4ba21 --- /dev/null +++ b/rendered/manifests/tests_enable_ui/splunk-connect-for-snmp/templates/redis/redis-standalone-statefulset.yaml @@ -0,0 +1,112 @@ +--- +# Source: splunk-connect-for-snmp/templates/redis/redis-standalone-statefulset.yaml +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: release-name-redis + namespace: default + labels: + app: release-name-redis +spec: + serviceName: release-name-redis + replicas: 1 + selector: + matchLabels: + app: release-name-redis + template: + metadata: + labels: + app: release-name-redis + annotations: + checksum/config: e82c09fa615350d9c147a0884485f953308babd6b8842d0cbe695ed5595eb530 + spec: + securityContext: + runAsUser: 999 + fsGroup: 999 + initContainers: + - name: fix-permissions + image: redis:8.2.2 + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - | + echo "=== Redis Init: Fixing Permissions ===" + echo "Current ownership:" + ls -ln /data + echo "" + echo "Fixing ownership to 999:999..." + chown -R 999:999 /data + chmod -R 755 /data + echo "" + echo "New ownership:" + ls -ln /data + echo "=== Permissions Fixed ===" + volumeMounts: + - name: redis-data + mountPath: /data + securityContext: + runAsUser: 0 # Must run as root to chown + containers: + - name: redis + image: redis:8.2.2 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 6379 + name: redis + command: + - sh + - -c + args: + - | + # Copy config to writable location + cp /etc/redis/redis.conf /tmp/redis.conf + + # Start Redis + exec redis-server /tmp/redis.conf + volumeMounts: + - name: redis-data + mountPath: /data + - name: redis-config + mountPath: /etc/redis + resources: + limits: + cpu: 500m + memory: 512Mi + requests: + cpu: 250m + memory: 256Mi + livenessProbe: + exec: + command: + - sh + - -c + - | + redis-cli ping + initialDelaySeconds: 30 + periodSeconds: 10 + readinessProbe: + exec: + command: + - sh + - -c + - | + redis-cli ping + initialDelaySeconds: 5 + periodSeconds: 5 + # Storage enabled but no existing PVC - use volumeClaimTemplates below + volumes: + - name: redis-config + configMap: + name: release-name-redis-config + # No existing PVC found, create new one via volumeClaimTemplates + volumeClaimTemplates: + - metadata: + name: redis-data + spec: + accessModes: + - ReadWriteOnce + storageClassName: microk8s-hostpath + resources: + requests: + storage: 5Gi diff --git a/rendered/manifests/tests_enable_ui/splunk-connect-for-snmp/templates/scheduler/deployment.yaml b/rendered/manifests/tests_enable_ui/splunk-connect-for-snmp/templates/scheduler/deployment.yaml index 40c223461..785864c76 100644 --- a/rendered/manifests/tests_enable_ui/splunk-connect-for-snmp/templates/scheduler/deployment.yaml +++ b/rendered/manifests/tests_enable_ui/splunk-connect-for-snmp/templates/scheduler/deployment.yaml @@ -44,10 +44,14 @@ spec: env: - name: CONFIG_PATH value: /app/config/config.yaml - - name: REDIS_URL - value: redis://release-name-redis-master:6379/1 - - name: CELERY_BROKER_URL - value: redis://release-name-redis-master:6379/0 + - name: REDIS_HOST + value: release-name-redis + - name: REDIS_PORT + value: "6379" + - name: REDIS_DB + value: "1" + - name: CELERY_DB + value: "0" - name: MONGO_URI value: mongodb://release-name-mongodb:27017 - name: MIB_SOURCES diff --git a/rendered/manifests/tests_enable_ui/splunk-connect-for-snmp/templates/traps/deployment.yaml b/rendered/manifests/tests_enable_ui/splunk-connect-for-snmp/templates/traps/deployment.yaml index 65c9b834f..db1be6303 100644 --- a/rendered/manifests/tests_enable_ui/splunk-connect-for-snmp/templates/traps/deployment.yaml +++ b/rendered/manifests/tests_enable_ui/splunk-connect-for-snmp/templates/traps/deployment.yaml @@ -44,10 +44,16 @@ spec: env: - name: CONFIG_PATH value: /app/config/config.yaml - - name: CELERY_BROKER_URL - value: redis://release-name-redis-master:6379/0 - name: MONGO_URI value: mongodb://release-name-mongodb:27017 + - name: REDIS_HOST + value: release-name-redis + - name: REDIS_PORT + value: "6379" + - name: REDIS_DB + value: "1" + - name: CELERY_DB + value: "0" - name: MIB_SOURCES value: "http://release-name-mibserver/asn1/@mib@" - name: MIB_INDEX diff --git a/rendered/manifests/tests_enable_ui/splunk-connect-for-snmp/templates/ui/configmap-backend.yaml b/rendered/manifests/tests_enable_ui/splunk-connect-for-snmp/templates/ui/configmap-backend.yaml index 97801f351..1d289ff4e 100644 --- a/rendered/manifests/tests_enable_ui/splunk-connect-for-snmp/templates/ui/configmap-backend.yaml +++ b/rendered/manifests/tests_enable_ui/splunk-connect-for-snmp/templates/ui/configmap-backend.yaml @@ -35,12 +35,16 @@ data: env: - name: CONFIG_PATH value: /app/config/config.yaml - - name: REDIS_URL - value: redis://release-name-redis-master:6379/1 + - name: REDIS_HOST + value: release-name-redis + - name: REDIS_PORT + value: "6379" + - name: REDIS_DB + value: "1" + - name: CELERY_DB + value: "0" - name: INVENTORY_PATH value: /app/inventory/inventory.csv - - name: CELERY_BROKER_URL - value: redis://release-name-redis-master:6379/0 - name: MONGO_URI value: mongodb://release-name-mongodb:27017 - name: MIB_SOURCES diff --git a/rendered/manifests/tests_enable_ui/splunk-connect-for-snmp/templates/ui/deployment-backend-worker.yaml b/rendered/manifests/tests_enable_ui/splunk-connect-for-snmp/templates/ui/deployment-backend-worker.yaml index da52f8b66..b00205e6f 100644 --- a/rendered/manifests/tests_enable_ui/splunk-connect-for-snmp/templates/ui/deployment-backend-worker.yaml +++ b/rendered/manifests/tests_enable_ui/splunk-connect-for-snmp/templates/ui/deployment-backend-worker.yaml @@ -24,14 +24,18 @@ spec: env: - name: MONGO_URI value: mongodb://release-name-mongodb:27017 - - name: REDIS_URL - value: redis://release-name-redis-master:6379/3 + - name: REDIS_HOST + value: release-name-redis + - name: REDIS_PORT + value: "6379" + - name: REDIS_DB + value: "1" + - name: CELERY_DB + value: "0" - name: JOB_CONFIG_PATH value: /config/job_config.yaml - name: JOB_NAMESPACE value: sc4snmp - - name: CELERY_BROKER_URL - value: redis://release-name-redis-master:6379/2 - name: VALUES_DIRECTORY value: /var/values_dir ports: diff --git a/rendered/manifests/tests_enable_ui/splunk-connect-for-snmp/templates/ui/deployment-backend.yaml b/rendered/manifests/tests_enable_ui/splunk-connect-for-snmp/templates/ui/deployment-backend.yaml index 130790bd5..f3db0cf6a 100644 --- a/rendered/manifests/tests_enable_ui/splunk-connect-for-snmp/templates/ui/deployment-backend.yaml +++ b/rendered/manifests/tests_enable_ui/splunk-connect-for-snmp/templates/ui/deployment-backend.yaml @@ -44,14 +44,18 @@ spec: env: - name: MONGO_URI value: mongodb://release-name-mongodb:27017 - - name: REDIS_URL - value: redis://release-name-redis-master:6379/3 + - name: REDIS_HOST + value: release-name-redis + - name: REDIS_PORT + value: "6379" + - name: REDIS_DB + value: "1" + - name: CELERY_DB + value: "0" - name: JOB_CONFIG_PATH value: /config/job_config.yaml - name: JOB_NAMESPACE value: sc4snmp - - name: CELERY_BROKER_URL - value: redis://release-name-redis-master:6379/2 - name: VALUES_DIRECTORY value: /var/values_dir - name: VALUES_FILE diff --git a/rendered/manifests/tests_enable_ui/splunk-connect-for-snmp/templates/worker/poller/deployment.yaml b/rendered/manifests/tests_enable_ui/splunk-connect-for-snmp/templates/worker/poller/deployment.yaml index 01dfdcb78..d5f4381ed 100644 --- a/rendered/manifests/tests_enable_ui/splunk-connect-for-snmp/templates/worker/poller/deployment.yaml +++ b/rendered/manifests/tests_enable_ui/splunk-connect-for-snmp/templates/worker/poller/deployment.yaml @@ -44,12 +44,16 @@ spec: env: - name: CONFIG_PATH value: /app/config/config.yaml - - name: REDIS_URL - value: redis://release-name-redis-master:6379/1 + - name: REDIS_HOST + value: release-name-redis + - name: REDIS_PORT + value: "6379" + - name: REDIS_DB + value: "1" + - name: CELERY_DB + value: "0" - name: SC4SNMP_VERSION value: CURRENT-VERSION - - name: CELERY_BROKER_URL - value: redis://release-name-redis-master:6379/0 - name: MONGO_URI value: mongodb://release-name-mongodb:27017 - name: WALK_RETRY_MAX_INTERVAL @@ -66,6 +70,8 @@ spec: value: "true" - name: UDP_CONNECTION_TIMEOUT value: "3" + - name: UDP_CONNECTION_RETRIES + value: "5" - name: MAX_OID_TO_PROCESS value: "70" - name: MAX_REPETITIONS diff --git a/rendered/manifests/tests_enable_ui/splunk-connect-for-snmp/templates/worker/sender/deployment.yaml b/rendered/manifests/tests_enable_ui/splunk-connect-for-snmp/templates/worker/sender/deployment.yaml index fe08ce4ed..7e2ec223b 100644 --- a/rendered/manifests/tests_enable_ui/splunk-connect-for-snmp/templates/worker/sender/deployment.yaml +++ b/rendered/manifests/tests_enable_ui/splunk-connect-for-snmp/templates/worker/sender/deployment.yaml @@ -44,12 +44,16 @@ spec: env: - name: CONFIG_PATH value: /app/config/config.yaml - - name: REDIS_URL - value: redis://release-name-redis-master:6379/1 + - name: REDIS_HOST + value: release-name-redis + - name: REDIS_PORT + value: "6379" + - name: REDIS_DB + value: "1" + - name: CELERY_DB + value: "0" - name: SC4SNMP_VERSION value: CURRENT-VERSION - - name: CELERY_BROKER_URL - value: redis://release-name-redis-master:6379/0 - name: MONGO_URI value: mongodb://release-name-mongodb:27017 - name: WALK_RETRY_MAX_INTERVAL @@ -66,6 +70,8 @@ spec: value: "true" - name: UDP_CONNECTION_TIMEOUT value: "3" + - name: UDP_CONNECTION_RETRIES + value: "5" - name: MAX_OID_TO_PROCESS value: "70" - name: MAX_REPETITIONS diff --git a/rendered/manifests/tests_enable_ui/splunk-connect-for-snmp/templates/worker/trap/deployment.yaml b/rendered/manifests/tests_enable_ui/splunk-connect-for-snmp/templates/worker/trap/deployment.yaml index 82675bad1..d506f7a42 100644 --- a/rendered/manifests/tests_enable_ui/splunk-connect-for-snmp/templates/worker/trap/deployment.yaml +++ b/rendered/manifests/tests_enable_ui/splunk-connect-for-snmp/templates/worker/trap/deployment.yaml @@ -44,12 +44,16 @@ spec: env: - name: CONFIG_PATH value: /app/config/config.yaml - - name: REDIS_URL - value: redis://release-name-redis-master:6379/1 + - name: REDIS_HOST + value: release-name-redis + - name: REDIS_PORT + value: "6379" + - name: REDIS_DB + value: "1" + - name: CELERY_DB + value: "0" - name: SC4SNMP_VERSION value: CURRENT-VERSION - - name: CELERY_BROKER_URL - value: redis://release-name-redis-master:6379/0 - name: MONGO_URI value: mongodb://release-name-mongodb:27017 - name: WALK_RETRY_MAX_INTERVAL @@ -66,6 +70,8 @@ spec: value: "true" - name: UDP_CONNECTION_TIMEOUT value: "3" + - name: UDP_CONNECTION_RETRIES + value: "5" - name: MAX_OID_TO_PROCESS value: "70" - name: MAX_REPETITIONS diff --git a/rendered/manifests/tests_metallb_false/splunk-connect-for-snmp/templates/common/discovery-config.yaml b/rendered/manifests/tests_metallb_false/splunk-connect-for-snmp/templates/common/discovery-config.yaml new file mode 100644 index 000000000..8f5acde96 --- /dev/null +++ b/rendered/manifests/tests_metallb_false/splunk-connect-for-snmp/templates/common/discovery-config.yaml @@ -0,0 +1,12 @@ +--- +# Source: splunk-connect-for-snmp/templates/common/discovery-config.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: splunk-connect-for-snmp-discovery-config +data: + discovery-config.yaml: | + enabled: false + ipv6Enabled: false + logLevel: INFO + usernameSecrets: [] diff --git a/rendered/manifests/tests_metallb_false/splunk-connect-for-snmp/templates/redis/redis-config.yaml b/rendered/manifests/tests_metallb_false/splunk-connect-for-snmp/templates/redis/redis-config.yaml new file mode 100644 index 000000000..22da33840 --- /dev/null +++ b/rendered/manifests/tests_metallb_false/splunk-connect-for-snmp/templates/redis/redis-config.yaml @@ -0,0 +1,33 @@ +--- +# Source: splunk-connect-for-snmp/templates/redis/redis-config.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: release-name-redis-config + namespace: default + labels: + app: release-name-redis +data: + redis.conf: | + # Data directory + dir /data + + # Persistence - RDB + save 900 1 + save 300 10 + save 60 10000 + + # Persistence - AOF + appendonly yes + appendfsync everysec + + # Logging + loglevel notice + + # Memory + maxmemory-policy noeviction + + # Network + bind 0.0.0.0 + protected-mode no + port 6379 diff --git a/rendered/manifests/tests_metallb_false/splunk-connect-for-snmp/templates/redis/redis-standalone-service.yaml b/rendered/manifests/tests_metallb_false/splunk-connect-for-snmp/templates/redis/redis-standalone-service.yaml new file mode 100644 index 000000000..5b3445437 --- /dev/null +++ b/rendered/manifests/tests_metallb_false/splunk-connect-for-snmp/templates/redis/redis-standalone-service.yaml @@ -0,0 +1,15 @@ +--- +# Source: splunk-connect-for-snmp/templates/redis/redis-standalone-service.yaml +apiVersion: v1 +kind: Service +metadata: + name: release-name-redis + namespace: default +spec: + type: ClusterIP + ports: + - port: 6379 + targetPort: 6379 + name: redis + selector: + app: release-name-redis diff --git a/rendered/manifests/tests_metallb_false/splunk-connect-for-snmp/templates/redis/redis-standalone-statefulset.yaml b/rendered/manifests/tests_metallb_false/splunk-connect-for-snmp/templates/redis/redis-standalone-statefulset.yaml new file mode 100644 index 000000000..b38e4ba21 --- /dev/null +++ b/rendered/manifests/tests_metallb_false/splunk-connect-for-snmp/templates/redis/redis-standalone-statefulset.yaml @@ -0,0 +1,112 @@ +--- +# Source: splunk-connect-for-snmp/templates/redis/redis-standalone-statefulset.yaml +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: release-name-redis + namespace: default + labels: + app: release-name-redis +spec: + serviceName: release-name-redis + replicas: 1 + selector: + matchLabels: + app: release-name-redis + template: + metadata: + labels: + app: release-name-redis + annotations: + checksum/config: e82c09fa615350d9c147a0884485f953308babd6b8842d0cbe695ed5595eb530 + spec: + securityContext: + runAsUser: 999 + fsGroup: 999 + initContainers: + - name: fix-permissions + image: redis:8.2.2 + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - | + echo "=== Redis Init: Fixing Permissions ===" + echo "Current ownership:" + ls -ln /data + echo "" + echo "Fixing ownership to 999:999..." + chown -R 999:999 /data + chmod -R 755 /data + echo "" + echo "New ownership:" + ls -ln /data + echo "=== Permissions Fixed ===" + volumeMounts: + - name: redis-data + mountPath: /data + securityContext: + runAsUser: 0 # Must run as root to chown + containers: + - name: redis + image: redis:8.2.2 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 6379 + name: redis + command: + - sh + - -c + args: + - | + # Copy config to writable location + cp /etc/redis/redis.conf /tmp/redis.conf + + # Start Redis + exec redis-server /tmp/redis.conf + volumeMounts: + - name: redis-data + mountPath: /data + - name: redis-config + mountPath: /etc/redis + resources: + limits: + cpu: 500m + memory: 512Mi + requests: + cpu: 250m + memory: 256Mi + livenessProbe: + exec: + command: + - sh + - -c + - | + redis-cli ping + initialDelaySeconds: 30 + periodSeconds: 10 + readinessProbe: + exec: + command: + - sh + - -c + - | + redis-cli ping + initialDelaySeconds: 5 + periodSeconds: 5 + # Storage enabled but no existing PVC - use volumeClaimTemplates below + volumes: + - name: redis-config + configMap: + name: release-name-redis-config + # No existing PVC found, create new one via volumeClaimTemplates + volumeClaimTemplates: + - metadata: + name: redis-data + spec: + accessModes: + - ReadWriteOnce + storageClassName: microk8s-hostpath + resources: + requests: + storage: 5Gi diff --git a/rendered/manifests/tests_metallb_false/splunk-connect-for-snmp/templates/traps/deployment.yaml b/rendered/manifests/tests_metallb_false/splunk-connect-for-snmp/templates/traps/deployment.yaml index 65c9b834f..db1be6303 100644 --- a/rendered/manifests/tests_metallb_false/splunk-connect-for-snmp/templates/traps/deployment.yaml +++ b/rendered/manifests/tests_metallb_false/splunk-connect-for-snmp/templates/traps/deployment.yaml @@ -44,10 +44,16 @@ spec: env: - name: CONFIG_PATH value: /app/config/config.yaml - - name: CELERY_BROKER_URL - value: redis://release-name-redis-master:6379/0 - name: MONGO_URI value: mongodb://release-name-mongodb:27017 + - name: REDIS_HOST + value: release-name-redis + - name: REDIS_PORT + value: "6379" + - name: REDIS_DB + value: "1" + - name: CELERY_DB + value: "0" - name: MIB_SOURCES value: "http://release-name-mibserver/asn1/@mib@" - name: MIB_INDEX diff --git a/rendered/manifests/tests_metallb_false/splunk-connect-for-snmp/templates/worker/sender/deployment.yaml b/rendered/manifests/tests_metallb_false/splunk-connect-for-snmp/templates/worker/sender/deployment.yaml index fe08ce4ed..7e2ec223b 100644 --- a/rendered/manifests/tests_metallb_false/splunk-connect-for-snmp/templates/worker/sender/deployment.yaml +++ b/rendered/manifests/tests_metallb_false/splunk-connect-for-snmp/templates/worker/sender/deployment.yaml @@ -44,12 +44,16 @@ spec: env: - name: CONFIG_PATH value: /app/config/config.yaml - - name: REDIS_URL - value: redis://release-name-redis-master:6379/1 + - name: REDIS_HOST + value: release-name-redis + - name: REDIS_PORT + value: "6379" + - name: REDIS_DB + value: "1" + - name: CELERY_DB + value: "0" - name: SC4SNMP_VERSION value: CURRENT-VERSION - - name: CELERY_BROKER_URL - value: redis://release-name-redis-master:6379/0 - name: MONGO_URI value: mongodb://release-name-mongodb:27017 - name: WALK_RETRY_MAX_INTERVAL @@ -66,6 +70,8 @@ spec: value: "true" - name: UDP_CONNECTION_TIMEOUT value: "3" + - name: UDP_CONNECTION_RETRIES + value: "5" - name: MAX_OID_TO_PROCESS value: "70" - name: MAX_REPETITIONS diff --git a/rendered/manifests/tests_metallb_false/splunk-connect-for-snmp/templates/worker/trap/deployment.yaml b/rendered/manifests/tests_metallb_false/splunk-connect-for-snmp/templates/worker/trap/deployment.yaml index 82675bad1..d506f7a42 100644 --- a/rendered/manifests/tests_metallb_false/splunk-connect-for-snmp/templates/worker/trap/deployment.yaml +++ b/rendered/manifests/tests_metallb_false/splunk-connect-for-snmp/templates/worker/trap/deployment.yaml @@ -44,12 +44,16 @@ spec: env: - name: CONFIG_PATH value: /app/config/config.yaml - - name: REDIS_URL - value: redis://release-name-redis-master:6379/1 + - name: REDIS_HOST + value: release-name-redis + - name: REDIS_PORT + value: "6379" + - name: REDIS_DB + value: "1" + - name: CELERY_DB + value: "0" - name: SC4SNMP_VERSION value: CURRENT-VERSION - - name: CELERY_BROKER_URL - value: redis://release-name-redis-master:6379/0 - name: MONGO_URI value: mongodb://release-name-mongodb:27017 - name: WALK_RETRY_MAX_INTERVAL @@ -66,6 +70,8 @@ spec: value: "true" - name: UDP_CONNECTION_TIMEOUT value: "3" + - name: UDP_CONNECTION_RETRIES + value: "5" - name: MAX_OID_TO_PROCESS value: "70" - name: MAX_REPETITIONS diff --git a/rendered/manifests/tests_mongodb_custom_image/splunk-connect-for-snmp/templates/common/discovery-config.yaml b/rendered/manifests/tests_mongodb_custom_image/splunk-connect-for-snmp/templates/common/discovery-config.yaml new file mode 100644 index 000000000..8f5acde96 --- /dev/null +++ b/rendered/manifests/tests_mongodb_custom_image/splunk-connect-for-snmp/templates/common/discovery-config.yaml @@ -0,0 +1,12 @@ +--- +# Source: splunk-connect-for-snmp/templates/common/discovery-config.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: splunk-connect-for-snmp-discovery-config +data: + discovery-config.yaml: | + enabled: false + ipv6Enabled: false + logLevel: INFO + usernameSecrets: [] diff --git a/rendered/manifests/tests_mongodb_custom_image/splunk-connect-for-snmp/templates/inventory/job.yaml b/rendered/manifests/tests_mongodb_custom_image/splunk-connect-for-snmp/templates/inventory/job.yaml index 3ea664cc0..2708b9ffd 100644 --- a/rendered/manifests/tests_mongodb_custom_image/splunk-connect-for-snmp/templates/inventory/job.yaml +++ b/rendered/manifests/tests_mongodb_custom_image/splunk-connect-for-snmp/templates/inventory/job.yaml @@ -28,12 +28,16 @@ spec: env: - name: CONFIG_PATH value: /app/config/config.yaml - - name: REDIS_URL - value: redis://release-name-redis-master:6379/1 + - name: REDIS_HOST + value: release-name-redis + - name: REDIS_PORT + value: "6379" + - name: REDIS_DB + value: "1" + - name: CELERY_DB + value: "0" - name: INVENTORY_PATH value: /app/inventory/inventory.csv - - name: CELERY_BROKER_URL - value: redis://release-name-redis-master:6379/0 - name: MONGO_URI value: mongodb://release-name-mongodb:27017 - name: MIB_SOURCES diff --git a/rendered/manifests/tests_mongodb_custom_image/splunk-connect-for-snmp/templates/redis/redis-config.yaml b/rendered/manifests/tests_mongodb_custom_image/splunk-connect-for-snmp/templates/redis/redis-config.yaml new file mode 100644 index 000000000..22da33840 --- /dev/null +++ b/rendered/manifests/tests_mongodb_custom_image/splunk-connect-for-snmp/templates/redis/redis-config.yaml @@ -0,0 +1,33 @@ +--- +# Source: splunk-connect-for-snmp/templates/redis/redis-config.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: release-name-redis-config + namespace: default + labels: + app: release-name-redis +data: + redis.conf: | + # Data directory + dir /data + + # Persistence - RDB + save 900 1 + save 300 10 + save 60 10000 + + # Persistence - AOF + appendonly yes + appendfsync everysec + + # Logging + loglevel notice + + # Memory + maxmemory-policy noeviction + + # Network + bind 0.0.0.0 + protected-mode no + port 6379 diff --git a/rendered/manifests/tests_mongodb_custom_image/splunk-connect-for-snmp/templates/redis/redis-standalone-service.yaml b/rendered/manifests/tests_mongodb_custom_image/splunk-connect-for-snmp/templates/redis/redis-standalone-service.yaml new file mode 100644 index 000000000..5b3445437 --- /dev/null +++ b/rendered/manifests/tests_mongodb_custom_image/splunk-connect-for-snmp/templates/redis/redis-standalone-service.yaml @@ -0,0 +1,15 @@ +--- +# Source: splunk-connect-for-snmp/templates/redis/redis-standalone-service.yaml +apiVersion: v1 +kind: Service +metadata: + name: release-name-redis + namespace: default +spec: + type: ClusterIP + ports: + - port: 6379 + targetPort: 6379 + name: redis + selector: + app: release-name-redis diff --git a/rendered/manifests/tests_mongodb_custom_image/splunk-connect-for-snmp/templates/redis/redis-standalone-statefulset.yaml b/rendered/manifests/tests_mongodb_custom_image/splunk-connect-for-snmp/templates/redis/redis-standalone-statefulset.yaml new file mode 100644 index 000000000..b38e4ba21 --- /dev/null +++ b/rendered/manifests/tests_mongodb_custom_image/splunk-connect-for-snmp/templates/redis/redis-standalone-statefulset.yaml @@ -0,0 +1,112 @@ +--- +# Source: splunk-connect-for-snmp/templates/redis/redis-standalone-statefulset.yaml +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: release-name-redis + namespace: default + labels: + app: release-name-redis +spec: + serviceName: release-name-redis + replicas: 1 + selector: + matchLabels: + app: release-name-redis + template: + metadata: + labels: + app: release-name-redis + annotations: + checksum/config: e82c09fa615350d9c147a0884485f953308babd6b8842d0cbe695ed5595eb530 + spec: + securityContext: + runAsUser: 999 + fsGroup: 999 + initContainers: + - name: fix-permissions + image: redis:8.2.2 + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - | + echo "=== Redis Init: Fixing Permissions ===" + echo "Current ownership:" + ls -ln /data + echo "" + echo "Fixing ownership to 999:999..." + chown -R 999:999 /data + chmod -R 755 /data + echo "" + echo "New ownership:" + ls -ln /data + echo "=== Permissions Fixed ===" + volumeMounts: + - name: redis-data + mountPath: /data + securityContext: + runAsUser: 0 # Must run as root to chown + containers: + - name: redis + image: redis:8.2.2 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 6379 + name: redis + command: + - sh + - -c + args: + - | + # Copy config to writable location + cp /etc/redis/redis.conf /tmp/redis.conf + + # Start Redis + exec redis-server /tmp/redis.conf + volumeMounts: + - name: redis-data + mountPath: /data + - name: redis-config + mountPath: /etc/redis + resources: + limits: + cpu: 500m + memory: 512Mi + requests: + cpu: 250m + memory: 256Mi + livenessProbe: + exec: + command: + - sh + - -c + - | + redis-cli ping + initialDelaySeconds: 30 + periodSeconds: 10 + readinessProbe: + exec: + command: + - sh + - -c + - | + redis-cli ping + initialDelaySeconds: 5 + periodSeconds: 5 + # Storage enabled but no existing PVC - use volumeClaimTemplates below + volumes: + - name: redis-config + configMap: + name: release-name-redis-config + # No existing PVC found, create new one via volumeClaimTemplates + volumeClaimTemplates: + - metadata: + name: redis-data + spec: + accessModes: + - ReadWriteOnce + storageClassName: microk8s-hostpath + resources: + requests: + storage: 5Gi diff --git a/rendered/manifests/tests_mongodb_custom_image/splunk-connect-for-snmp/templates/scheduler/deployment.yaml b/rendered/manifests/tests_mongodb_custom_image/splunk-connect-for-snmp/templates/scheduler/deployment.yaml index 40c223461..785864c76 100644 --- a/rendered/manifests/tests_mongodb_custom_image/splunk-connect-for-snmp/templates/scheduler/deployment.yaml +++ b/rendered/manifests/tests_mongodb_custom_image/splunk-connect-for-snmp/templates/scheduler/deployment.yaml @@ -44,10 +44,14 @@ spec: env: - name: CONFIG_PATH value: /app/config/config.yaml - - name: REDIS_URL - value: redis://release-name-redis-master:6379/1 - - name: CELERY_BROKER_URL - value: redis://release-name-redis-master:6379/0 + - name: REDIS_HOST + value: release-name-redis + - name: REDIS_PORT + value: "6379" + - name: REDIS_DB + value: "1" + - name: CELERY_DB + value: "0" - name: MONGO_URI value: mongodb://release-name-mongodb:27017 - name: MIB_SOURCES diff --git a/rendered/manifests/tests_mongodb_custom_image/splunk-connect-for-snmp/templates/traps/deployment.yaml b/rendered/manifests/tests_mongodb_custom_image/splunk-connect-for-snmp/templates/traps/deployment.yaml index 65c9b834f..db1be6303 100644 --- a/rendered/manifests/tests_mongodb_custom_image/splunk-connect-for-snmp/templates/traps/deployment.yaml +++ b/rendered/manifests/tests_mongodb_custom_image/splunk-connect-for-snmp/templates/traps/deployment.yaml @@ -44,10 +44,16 @@ spec: env: - name: CONFIG_PATH value: /app/config/config.yaml - - name: CELERY_BROKER_URL - value: redis://release-name-redis-master:6379/0 - name: MONGO_URI value: mongodb://release-name-mongodb:27017 + - name: REDIS_HOST + value: release-name-redis + - name: REDIS_PORT + value: "6379" + - name: REDIS_DB + value: "1" + - name: CELERY_DB + value: "0" - name: MIB_SOURCES value: "http://release-name-mibserver/asn1/@mib@" - name: MIB_INDEX diff --git a/rendered/manifests/tests_mongodb_custom_image/splunk-connect-for-snmp/templates/worker/poller/deployment.yaml b/rendered/manifests/tests_mongodb_custom_image/splunk-connect-for-snmp/templates/worker/poller/deployment.yaml index 01dfdcb78..d5f4381ed 100644 --- a/rendered/manifests/tests_mongodb_custom_image/splunk-connect-for-snmp/templates/worker/poller/deployment.yaml +++ b/rendered/manifests/tests_mongodb_custom_image/splunk-connect-for-snmp/templates/worker/poller/deployment.yaml @@ -44,12 +44,16 @@ spec: env: - name: CONFIG_PATH value: /app/config/config.yaml - - name: REDIS_URL - value: redis://release-name-redis-master:6379/1 + - name: REDIS_HOST + value: release-name-redis + - name: REDIS_PORT + value: "6379" + - name: REDIS_DB + value: "1" + - name: CELERY_DB + value: "0" - name: SC4SNMP_VERSION value: CURRENT-VERSION - - name: CELERY_BROKER_URL - value: redis://release-name-redis-master:6379/0 - name: MONGO_URI value: mongodb://release-name-mongodb:27017 - name: WALK_RETRY_MAX_INTERVAL @@ -66,6 +70,8 @@ spec: value: "true" - name: UDP_CONNECTION_TIMEOUT value: "3" + - name: UDP_CONNECTION_RETRIES + value: "5" - name: MAX_OID_TO_PROCESS value: "70" - name: MAX_REPETITIONS diff --git a/rendered/manifests/tests_mongodb_custom_image/splunk-connect-for-snmp/templates/worker/sender/deployment.yaml b/rendered/manifests/tests_mongodb_custom_image/splunk-connect-for-snmp/templates/worker/sender/deployment.yaml index fe08ce4ed..7e2ec223b 100644 --- a/rendered/manifests/tests_mongodb_custom_image/splunk-connect-for-snmp/templates/worker/sender/deployment.yaml +++ b/rendered/manifests/tests_mongodb_custom_image/splunk-connect-for-snmp/templates/worker/sender/deployment.yaml @@ -44,12 +44,16 @@ spec: env: - name: CONFIG_PATH value: /app/config/config.yaml - - name: REDIS_URL - value: redis://release-name-redis-master:6379/1 + - name: REDIS_HOST + value: release-name-redis + - name: REDIS_PORT + value: "6379" + - name: REDIS_DB + value: "1" + - name: CELERY_DB + value: "0" - name: SC4SNMP_VERSION value: CURRENT-VERSION - - name: CELERY_BROKER_URL - value: redis://release-name-redis-master:6379/0 - name: MONGO_URI value: mongodb://release-name-mongodb:27017 - name: WALK_RETRY_MAX_INTERVAL @@ -66,6 +70,8 @@ spec: value: "true" - name: UDP_CONNECTION_TIMEOUT value: "3" + - name: UDP_CONNECTION_RETRIES + value: "5" - name: MAX_OID_TO_PROCESS value: "70" - name: MAX_REPETITIONS diff --git a/rendered/manifests/tests_mongodb_custom_image/splunk-connect-for-snmp/templates/worker/trap/deployment.yaml b/rendered/manifests/tests_mongodb_custom_image/splunk-connect-for-snmp/templates/worker/trap/deployment.yaml index 82675bad1..d506f7a42 100644 --- a/rendered/manifests/tests_mongodb_custom_image/splunk-connect-for-snmp/templates/worker/trap/deployment.yaml +++ b/rendered/manifests/tests_mongodb_custom_image/splunk-connect-for-snmp/templates/worker/trap/deployment.yaml @@ -44,12 +44,16 @@ spec: env: - name: CONFIG_PATH value: /app/config/config.yaml - - name: REDIS_URL - value: redis://release-name-redis-master:6379/1 + - name: REDIS_HOST + value: release-name-redis + - name: REDIS_PORT + value: "6379" + - name: REDIS_DB + value: "1" + - name: CELERY_DB + value: "0" - name: SC4SNMP_VERSION value: CURRENT-VERSION - - name: CELERY_BROKER_URL - value: redis://release-name-redis-master:6379/0 - name: MONGO_URI value: mongodb://release-name-mongodb:27017 - name: WALK_RETRY_MAX_INTERVAL @@ -66,6 +70,8 @@ spec: value: "true" - name: UDP_CONNECTION_TIMEOUT value: "3" + - name: UDP_CONNECTION_RETRIES + value: "5" - name: MAX_OID_TO_PROCESS value: "70" - name: MAX_REPETITIONS diff --git a/rendered/manifests/tests_only_discovery/splunk-connect-for-snmp/templates/common/discovery-config.yaml b/rendered/manifests/tests_only_discovery/splunk-connect-for-snmp/templates/common/discovery-config.yaml new file mode 100644 index 000000000..fb9e937c0 --- /dev/null +++ b/rendered/manifests/tests_only_discovery/splunk-connect-for-snmp/templates/common/discovery-config.yaml @@ -0,0 +1,41 @@ +--- +# Source: splunk-connect-for-snmp/templates/common/discovery-config.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: splunk-connect-for-snmp-discovery-config +data: + discovery-config.yaml: | + autodiscovery: + discovery_version2c: + community: public + delete_already_discovered: false + device_rules: + - group: linux_group + name: Linux servers + patterns: '*linux*' + - group: centos_group + name: Centos servers + patterns: '*centos*' + frequency: 21600 + network_address: 4.91.99.113/28 + port: 161 + version: 2c + discovery_version3: + delete_already_discovered: false + device_rules: + - group: linux_group + name: Linux VM + patterns: '*Linux*' + frequency: 21600 + network_address: 4.91.99.113/28 + port: 161 + secret: secret + security_engine: 80001f8880e761866965756b6800000000 + version: "3" + discoveryPath: /home/devuser/discovery-test2 + enabled: true + ipv6Enabled: true + logLevel: INFO + usernameSecrets: + - secret diff --git a/rendered/manifests/tests_only_discovery/splunk-connect-for-snmp/templates/common/scheduler-config.yaml b/rendered/manifests/tests_only_discovery/splunk-connect-for-snmp/templates/common/scheduler-config.yaml new file mode 100644 index 000000000..2ace18f92 --- /dev/null +++ b/rendered/manifests/tests_only_discovery/splunk-connect-for-snmp/templates/common/scheduler-config.yaml @@ -0,0 +1,21 @@ +--- +# Source: splunk-connect-for-snmp/templates/common/scheduler-config.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: splunk-connect-for-snmp-config + labels: + app.kubernetes.io/name: splunk-connect-for-snmp-scheduler + app.kubernetes.io/instance: release-name + helm.sh/chart: splunk-connect-for-snmp-CURRENT-VERSION + app.kubernetes.io/version: "CURRENT-VERSION" + app.kubernetes.io/managed-by: Helm +data: + config.yaml: |- + communities: + public: + communityIndex: + contextEngineId: + contextName: + tag: + securityName: diff --git a/rendered/manifests/tests_only_discovery/splunk-connect-for-snmp/templates/common/splunk-secret.yaml b/rendered/manifests/tests_only_discovery/splunk-connect-for-snmp/templates/common/splunk-secret.yaml new file mode 100644 index 000000000..21e689f0a --- /dev/null +++ b/rendered/manifests/tests_only_discovery/splunk-connect-for-snmp/templates/common/splunk-secret.yaml @@ -0,0 +1,9 @@ +--- +# Source: splunk-connect-for-snmp/templates/common/splunk-secret.yaml +apiVersion: v1 +kind: Secret +metadata: + name: splunk-connect-for-snmp-splunk +type: Opaque +data: + hec_token: "MDAwMDAwMDAtMDAwMC0wMDAwLTAwMDAtMDAwMDAwMDAwMDAw" diff --git a/rendered/manifests/tests_only_discovery/splunk-connect-for-snmp/templates/discovery/job.yaml b/rendered/manifests/tests_only_discovery/splunk-connect-for-snmp/templates/discovery/job.yaml new file mode 100644 index 000000000..245f36f10 --- /dev/null +++ b/rendered/manifests/tests_only_discovery/splunk-connect-for-snmp/templates/discovery/job.yaml @@ -0,0 +1,64 @@ +--- +# Source: splunk-connect-for-snmp/templates/discovery/job.yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: release-name-splunk-connect-for-snmp-discovery + labels: + app.kubernetes.io/name: splunk-connect-for-snmp-discovery + app.kubernetes.io/instance: release-name + helm.sh/chart: splunk-connect-for-snmp-CURRENT-VERSION + app.kubernetes.io/version: "CURRENT-VERSION" + app.kubernetes.io/managed-by: Helm +spec: + ttlSecondsAfterFinished: 300 + template: + metadata: + # + + labels: + app.kubernetes.io/name: splunk-connect-for-snmp-discovery + app.kubernetes.io/instance: release-name + spec: + containers: + - name: splunk-connect-for-snmp-discovery + image: "ghcr.io/splunk/splunk-connect-for-snmp/container:CURRENT-VERSION" + imagePullPolicy: Always + args: + ["discovery"] + env: + - name: REDIS_HOST + value: release-name-redis + - name: REDIS_PORT + value: "6379" + - name: REDIS_DB + value: "1" + - name: CELERY_DB + value: "0" + - name: DISCOVERY_CONFIG_PATH + value: /app/discovery/discovery-config.yaml + - name: LOG_LEVEL + value: INFO + - name: CHAIN_OF_TASKS_EXPIRY_TIME + value: "60" + - name: CELERY_TASK_TIMEOUT + value: "2400" + volumeMounts: + - name: discovery-config + mountPath: "/app/discovery" + readOnly: true + - name: tmp + mountPath: "/tmp/" + readOnly: false + + volumes: + # # You set volumes at the Pod level, then mount them into containers inside that Pod + - name: discovery-config + configMap: + name: splunk-connect-for-snmp-discovery-config + items: + - key: "discovery-config.yaml" + path: "discovery-config.yaml" + - name: tmp + emptyDir: {} + restartPolicy: OnFailure diff --git a/rendered/manifests/tests_only_discovery/splunk-connect-for-snmp/templates/redis/redis-config.yaml b/rendered/manifests/tests_only_discovery/splunk-connect-for-snmp/templates/redis/redis-config.yaml new file mode 100644 index 000000000..22da33840 --- /dev/null +++ b/rendered/manifests/tests_only_discovery/splunk-connect-for-snmp/templates/redis/redis-config.yaml @@ -0,0 +1,33 @@ +--- +# Source: splunk-connect-for-snmp/templates/redis/redis-config.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: release-name-redis-config + namespace: default + labels: + app: release-name-redis +data: + redis.conf: | + # Data directory + dir /data + + # Persistence - RDB + save 900 1 + save 300 10 + save 60 10000 + + # Persistence - AOF + appendonly yes + appendfsync everysec + + # Logging + loglevel notice + + # Memory + maxmemory-policy noeviction + + # Network + bind 0.0.0.0 + protected-mode no + port 6379 diff --git a/rendered/manifests/tests_only_discovery/splunk-connect-for-snmp/templates/redis/redis-standalone-service.yaml b/rendered/manifests/tests_only_discovery/splunk-connect-for-snmp/templates/redis/redis-standalone-service.yaml new file mode 100644 index 000000000..5b3445437 --- /dev/null +++ b/rendered/manifests/tests_only_discovery/splunk-connect-for-snmp/templates/redis/redis-standalone-service.yaml @@ -0,0 +1,15 @@ +--- +# Source: splunk-connect-for-snmp/templates/redis/redis-standalone-service.yaml +apiVersion: v1 +kind: Service +metadata: + name: release-name-redis + namespace: default +spec: + type: ClusterIP + ports: + - port: 6379 + targetPort: 6379 + name: redis + selector: + app: release-name-redis diff --git a/rendered/manifests/tests_only_discovery/splunk-connect-for-snmp/templates/redis/redis-standalone-statefulset.yaml b/rendered/manifests/tests_only_discovery/splunk-connect-for-snmp/templates/redis/redis-standalone-statefulset.yaml new file mode 100644 index 000000000..b38e4ba21 --- /dev/null +++ b/rendered/manifests/tests_only_discovery/splunk-connect-for-snmp/templates/redis/redis-standalone-statefulset.yaml @@ -0,0 +1,112 @@ +--- +# Source: splunk-connect-for-snmp/templates/redis/redis-standalone-statefulset.yaml +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: release-name-redis + namespace: default + labels: + app: release-name-redis +spec: + serviceName: release-name-redis + replicas: 1 + selector: + matchLabels: + app: release-name-redis + template: + metadata: + labels: + app: release-name-redis + annotations: + checksum/config: e82c09fa615350d9c147a0884485f953308babd6b8842d0cbe695ed5595eb530 + spec: + securityContext: + runAsUser: 999 + fsGroup: 999 + initContainers: + - name: fix-permissions + image: redis:8.2.2 + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - | + echo "=== Redis Init: Fixing Permissions ===" + echo "Current ownership:" + ls -ln /data + echo "" + echo "Fixing ownership to 999:999..." + chown -R 999:999 /data + chmod -R 755 /data + echo "" + echo "New ownership:" + ls -ln /data + echo "=== Permissions Fixed ===" + volumeMounts: + - name: redis-data + mountPath: /data + securityContext: + runAsUser: 0 # Must run as root to chown + containers: + - name: redis + image: redis:8.2.2 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 6379 + name: redis + command: + - sh + - -c + args: + - | + # Copy config to writable location + cp /etc/redis/redis.conf /tmp/redis.conf + + # Start Redis + exec redis-server /tmp/redis.conf + volumeMounts: + - name: redis-data + mountPath: /data + - name: redis-config + mountPath: /etc/redis + resources: + limits: + cpu: 500m + memory: 512Mi + requests: + cpu: 250m + memory: 256Mi + livenessProbe: + exec: + command: + - sh + - -c + - | + redis-cli ping + initialDelaySeconds: 30 + periodSeconds: 10 + readinessProbe: + exec: + command: + - sh + - -c + - | + redis-cli ping + initialDelaySeconds: 5 + periodSeconds: 5 + # Storage enabled but no existing PVC - use volumeClaimTemplates below + volumes: + - name: redis-config + configMap: + name: release-name-redis-config + # No existing PVC found, create new one via volumeClaimTemplates + volumeClaimTemplates: + - metadata: + name: redis-data + spec: + accessModes: + - ReadWriteOnce + storageClassName: microk8s-hostpath + resources: + requests: + storage: 5Gi diff --git a/rendered/manifests/tests_only_discovery/splunk-connect-for-snmp/templates/serviceaccount.yaml b/rendered/manifests/tests_only_discovery/splunk-connect-for-snmp/templates/serviceaccount.yaml new file mode 100644 index 000000000..59ae809f1 --- /dev/null +++ b/rendered/manifests/tests_only_discovery/splunk-connect-for-snmp/templates/serviceaccount.yaml @@ -0,0 +1,10 @@ +--- +# Source: splunk-connect-for-snmp/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: release-name-splunk-connect-for-snmp-user + labels: + helm.sh/chart: splunk-connect-for-snmp-CURRENT-VERSION + app.kubernetes.io/version: "CURRENT-VERSION" + app.kubernetes.io/managed-by: Helm diff --git a/rendered/manifests/tests_only_discovery/splunk-connect-for-snmp/templates/sim/pdb.yaml b/rendered/manifests/tests_only_discovery/splunk-connect-for-snmp/templates/sim/pdb.yaml new file mode 100644 index 000000000..0f1827e83 --- /dev/null +++ b/rendered/manifests/tests_only_discovery/splunk-connect-for-snmp/templates/sim/pdb.yaml @@ -0,0 +1,18 @@ +--- +# Source: splunk-connect-for-snmp/templates/sim/pdb.yaml +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: release-name-splunk-connect-for-snmp-sim + labels: + app.kubernetes.io/name: splunk-connect-for-snmp-sim + app.kubernetes.io/instance: release-name + helm.sh/chart: splunk-connect-for-snmp-CURRENT-VERSION + app.kubernetes.io/version: "CURRENT-VERSION" + app.kubernetes.io/managed-by: Helm +spec: + minAvailable: 80% + selector: + matchLabels: + app.kubernetes.io/name: splunk-connect-for-snmp-sim + app.kubernetes.io/instance: release-name diff --git a/rendered/manifests/tests_only_discovery/splunk-connect-for-snmp/templates/tests/test-connection.yaml b/rendered/manifests/tests_only_discovery/splunk-connect-for-snmp/templates/tests/test-connection.yaml new file mode 100644 index 000000000..6851a86ec --- /dev/null +++ b/rendered/manifests/tests_only_discovery/splunk-connect-for-snmp/templates/tests/test-connection.yaml @@ -0,0 +1,35 @@ +--- +# Source: splunk-connect-for-snmp/templates/tests/test-connection.yaml +apiVersion: v1 +kind: Pod +metadata: + name: "release-name-splunk-connect-for-snmp-trap-test-connection" + labels: + helm.sh/chart: splunk-connect-for-snmp-CURRENT-VERSION + app.kubernetes.io/version: "CURRENT-VERSION" + app.kubernetes.io/managed-by: Helm + annotations: + "helm.sh/hook": test + "kube-score/ignore": "pod-probes,pod-networkpolicy" +spec: + containers: + - name: wget + image: busybox:1.34.1 + imagePullPolicy: Always + command: ['wget'] + args: ['release-name-splunk-connect-for-snmp-trap:162'] + securityContext: + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 10001 + runAsGroup: 10001 + resources: + limits: + memory: 128Mi + requests: + cpu: 100m + memory: 128Mi + restartPolicy: Never diff --git a/rendered/manifests/tests_only_discovery/splunk-connect-for-snmp/templates/worker/discovery/deployment.yaml b/rendered/manifests/tests_only_discovery/splunk-connect-for-snmp/templates/worker/discovery/deployment.yaml new file mode 100644 index 000000000..e266d9a61 --- /dev/null +++ b/rendered/manifests/tests_only_discovery/splunk-connect-for-snmp/templates/worker/discovery/deployment.yaml @@ -0,0 +1,158 @@ +--- +# Source: splunk-connect-for-snmp/templates/worker/discovery/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: release-name-splunk-connect-for-snmp-worker-discovery + labels: + app.kubernetes.io/name: splunk-connect-for-snmp-worker-discovery + app.kubernetes.io/instance: release-name + helm.sh/chart: splunk-connect-for-snmp-CURRENT-VERSION + app.kubernetes.io/version: "CURRENT-VERSION" + app.kubernetes.io/managed-by: Helm +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: splunk-connect-for-snmp-worker-discovery + app.kubernetes.io/instance: release-name + template: + metadata: + labels: + app.kubernetes.io/name: splunk-connect-for-snmp-worker-discovery + app.kubernetes.io/instance: release-name + spec: + serviceAccountName: release-name-splunk-connect-for-snmp-user + securityContext: + fsGroup: 10001 + containers: + - name: splunk-connect-for-snmp-discovery + securityContext: + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 10001 + runAsGroup: 10001 + image: "ghcr.io/splunk/splunk-connect-for-snmp/container:CURRENT-VERSION" + imagePullPolicy: Always + args: + [ + "celery", "worker-discovery", + ] + env: + - name: CONFIG_PATH + value: /app/config/config.yaml + - name: REDIS_HOST + value: release-name-redis + - name: REDIS_PORT + value: "6379" + - name: REDIS_DB + value: "1" + - name: CELERY_DB + value: "0" + - name: SC4SNMP_VERSION + value: CURRENT-VERSION + - name: MONGO_URI + value: mongodb://release-name-mongodb:27017 + - name: WALK_RETRY_MAX_INTERVAL + value: "180" + - name: WALK_MAX_RETRIES + value: "5" + - name: METRICS_INDEXING_ENABLED + value: "false" + - name: POLL_BASE_PROFILES + value: "true" + - name: LOG_LEVEL + value: INFO + - name: DISABLE_MONGO_DEBUG_LOGGING + value: "true" + - name: UDP_CONNECTION_TIMEOUT + value: "3" + - name: UDP_CONNECTION_RETRIES + value: "5" + - name: MAX_OID_TO_PROCESS + value: "70" + - name: MAX_REPETITIONS + value: "10" + - name: PYSNMP_DEBUG + value: "" + - name: PROFILES_RELOAD_DELAY + value: "60" + - name: MIB_SOURCES + value: "http://release-name-mibserver/asn1/@mib@" + - name: MIB_INDEX + value: "http://release-name-mibserver/index.csv" + - name: MIB_STANDARD + value: "http://release-name-mibserver/standard.txt" + - name: SPLUNK_HEC_HOST + value: "" + - name: IGNORE_EMPTY_VARBINDS + value: "false" + - name: SPLUNK_HEC_PORT + value: "8088" + - name: SPLUNK_HEC_INSECURESSL + value: "false" + - name: SPLUNK_AGGREGATE_TRAPS_EVENTS + value: "false" + - name: SPLUNK_METRIC_NAME_HYPHEN_TO_UNDERSCORE + value: "false" + - name: SPLUNK_HEC_TOKEN + valueFrom: + secretKeyRef: + name: splunk-connect-for-snmp-splunk + key: hec_token + - name: SPLUNK_HEC_INDEX_EVENTS + value: netops + - name: SPLUNK_HEC_INDEX_METRICS + value: netmetrics + - name: SPLUNK_SOURCETYPE_TRAPS + value: "sc4snmp:traps" + - name: SPLUNK_SOURCETYPE_POLLING_EVENTS + value: "sc4snmp:event" + - name: SPLUNK_SOURCETYPE_POLLING_METRICS + value: "sc4snmp:metric" + - name: DISCOVERY_FOLDER_PATH + value: /app/discovery + - name: CELERY_TASK_TIMEOUT + value: "2400" + - name: IPv6_ENABLED + value: "true" + + volumeMounts: + - name: config + mountPath: "/app/config" + readOnly: true + - name: pysnmp-cache-volume + mountPath: "/.pysnmp/" + readOnly: false + - name: tmp + mountPath: "/tmp/" + readOnly: false + - name: discovery-volume + mountPath: /app/discovery + - name: secret-snmpv3-secrets + mountPath: /app/secrets/snmpv3/secret + readOnly: true + volumes: + # You set volumes at the Pod level, then mount them into containers inside that Pod + - name: config + configMap: + # Provide the name of the ConfigMap you want to mount. + name: splunk-connect-for-snmp-config + # An array of keys from the ConfigMap to create as files + items: + - key: "config.yaml" + path: "config.yaml" + - name: pysnmp-cache-volume + emptyDir: {} + - name: tmp + emptyDir: {} + - name: discovery-volume + hostPath: + path: /home/devuser/discovery-test2 + type: Directory + - name: secret-snmpv3-secrets + secret: + secretName: secret diff --git a/rendered/manifests/tests_only_discovery/splunk-connect-for-snmp/templates/worker/pdb.yaml b/rendered/manifests/tests_only_discovery/splunk-connect-for-snmp/templates/worker/pdb.yaml new file mode 100644 index 000000000..4b3ea594c --- /dev/null +++ b/rendered/manifests/tests_only_discovery/splunk-connect-for-snmp/templates/worker/pdb.yaml @@ -0,0 +1,18 @@ +--- +# Source: splunk-connect-for-snmp/templates/worker/pdb.yaml +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: release-name-splunk-connect-for-snmp-worker + labels: + app.kubernetes.io/name: splunk-connect-for-snmp-worker + app.kubernetes.io/instance: release-name + helm.sh/chart: splunk-connect-for-snmp-CURRENT-VERSION + app.kubernetes.io/version: "CURRENT-VERSION" + app.kubernetes.io/managed-by: Helm +spec: + minAvailable: 80% + selector: + matchLabels: + app.kubernetes.io/name: splunk-connect-for-snmp-worker + app.kubernetes.io/instance: release-name diff --git a/rendered/manifests/tests_only_discovery/splunk-connect-for-snmp/templates/worker/sender/deployment.yaml b/rendered/manifests/tests_only_discovery/splunk-connect-for-snmp/templates/worker/sender/deployment.yaml new file mode 100644 index 000000000..62417e8fa --- /dev/null +++ b/rendered/manifests/tests_only_discovery/splunk-connect-for-snmp/templates/worker/sender/deployment.yaml @@ -0,0 +1,158 @@ +--- +# Source: splunk-connect-for-snmp/templates/worker/sender/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: release-name-splunk-connect-for-snmp-worker-sender + labels: + app.kubernetes.io/name: splunk-connect-for-snmp-worker-sender + app.kubernetes.io/instance: release-name + helm.sh/chart: splunk-connect-for-snmp-CURRENT-VERSION + app.kubernetes.io/version: "CURRENT-VERSION" + app.kubernetes.io/managed-by: Helm +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: splunk-connect-for-snmp-worker-sender + app.kubernetes.io/instance: release-name + template: + metadata: + labels: + app.kubernetes.io/name: splunk-connect-for-snmp-worker-sender + app.kubernetes.io/instance: release-name + spec: + serviceAccountName: release-name-splunk-connect-for-snmp-user + securityContext: + fsGroup: 10001 + containers: + - name: splunk-connect-for-snmp-worker-sender + securityContext: + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 10001 + runAsGroup: 10001 + image: "ghcr.io/splunk/splunk-connect-for-snmp/container:CURRENT-VERSION" + imagePullPolicy: Always + args: + [ + "celery", "worker-sender", + ] + env: + - name: CONFIG_PATH + value: /app/config/config.yaml + - name: REDIS_HOST + value: release-name-redis + - name: REDIS_PORT + value: "6379" + - name: REDIS_DB + value: "1" + - name: CELERY_DB + value: "0" + - name: SC4SNMP_VERSION + value: CURRENT-VERSION + - name: MONGO_URI + value: mongodb://release-name-mongodb:27017 + - name: WALK_RETRY_MAX_INTERVAL + value: "180" + - name: WALK_MAX_RETRIES + value: "5" + - name: METRICS_INDEXING_ENABLED + value: "false" + - name: POLL_BASE_PROFILES + value: "true" + - name: LOG_LEVEL + value: INFO + - name: DISABLE_MONGO_DEBUG_LOGGING + value: "true" + - name: UDP_CONNECTION_TIMEOUT + value: "3" + - name: UDP_CONNECTION_RETRIES + value: "5" + - name: MAX_OID_TO_PROCESS + value: "70" + - name: MAX_REPETITIONS + value: "10" + - name: PYSNMP_DEBUG + value: "" + - name: PROFILES_RELOAD_DELAY + value: "60" + - name: MIB_SOURCES + value: "http://release-name-mibserver/asn1/@mib@" + - name: MIB_INDEX + value: "http://release-name-mibserver/index.csv" + - name: MIB_STANDARD + value: "http://release-name-mibserver/standard.txt" + - name: SPLUNK_HEC_HOST + value: "" + - name: IGNORE_EMPTY_VARBINDS + value: "false" + - name: SPLUNK_HEC_PORT + value: "8088" + - name: SPLUNK_HEC_INSECURESSL + value: "false" + - name: SPLUNK_AGGREGATE_TRAPS_EVENTS + value: "false" + - name: SPLUNK_METRIC_NAME_HYPHEN_TO_UNDERSCORE + value: "false" + - name: SPLUNK_HEC_TOKEN + valueFrom: + secretKeyRef: + name: splunk-connect-for-snmp-splunk + key: hec_token + - name: SPLUNK_HEC_INDEX_EVENTS + value: netops + - name: SPLUNK_HEC_INDEX_METRICS + value: netmetrics + - name: SPLUNK_SOURCETYPE_TRAPS + value: "sc4snmp:traps" + - name: SPLUNK_SOURCETYPE_POLLING_EVENTS + value: "sc4snmp:event" + - name: SPLUNK_SOURCETYPE_POLLING_METRICS + value: "sc4snmp:metric" + - name: WORKER_CONCURRENCY + value: "4" + - name: PREFETCH_COUNT + value: "30" + volumeMounts: + - name: config + mountPath: "/app/config" + readOnly: true + - name: pysnmp-cache-volume + mountPath: "/.pysnmp/" + readOnly: false + - name: tmp + mountPath: "/tmp/" + readOnly: false + resources: + limits: + cpu: 500m + requests: + cpu: 250m + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 1 + podAffinityTerm: + topologyKey: kubernetes.io/hostname + labelSelector: + matchLabels: + app.kubernetes.io/name: splunk-connect-for-snmp-worker-sender + app.kubernetes.io/instance: release-name + volumes: + # You set volumes at the Pod level, then mount them into containers inside that Pod + - name: config + configMap: + # Provide the name of the ConfigMap you want to mount. + name: splunk-connect-for-snmp-config + # An array of keys from the ConfigMap to create as files + items: + - key: "config.yaml" + path: "config.yaml" + - name: pysnmp-cache-volume + emptyDir: {} + - name: tmp + emptyDir: {} diff --git a/rendered/manifests/tests_only_polling/splunk-connect-for-snmp/templates/common/discovery-config.yaml b/rendered/manifests/tests_only_polling/splunk-connect-for-snmp/templates/common/discovery-config.yaml new file mode 100644 index 000000000..8f5acde96 --- /dev/null +++ b/rendered/manifests/tests_only_polling/splunk-connect-for-snmp/templates/common/discovery-config.yaml @@ -0,0 +1,12 @@ +--- +# Source: splunk-connect-for-snmp/templates/common/discovery-config.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: splunk-connect-for-snmp-discovery-config +data: + discovery-config.yaml: | + enabled: false + ipv6Enabled: false + logLevel: INFO + usernameSecrets: [] diff --git a/rendered/manifests/tests_only_polling/splunk-connect-for-snmp/templates/inventory/job.yaml b/rendered/manifests/tests_only_polling/splunk-connect-for-snmp/templates/inventory/job.yaml index 3ea664cc0..2708b9ffd 100644 --- a/rendered/manifests/tests_only_polling/splunk-connect-for-snmp/templates/inventory/job.yaml +++ b/rendered/manifests/tests_only_polling/splunk-connect-for-snmp/templates/inventory/job.yaml @@ -28,12 +28,16 @@ spec: env: - name: CONFIG_PATH value: /app/config/config.yaml - - name: REDIS_URL - value: redis://release-name-redis-master:6379/1 + - name: REDIS_HOST + value: release-name-redis + - name: REDIS_PORT + value: "6379" + - name: REDIS_DB + value: "1" + - name: CELERY_DB + value: "0" - name: INVENTORY_PATH value: /app/inventory/inventory.csv - - name: CELERY_BROKER_URL - value: redis://release-name-redis-master:6379/0 - name: MONGO_URI value: mongodb://release-name-mongodb:27017 - name: MIB_SOURCES diff --git a/rendered/manifests/tests_only_polling/splunk-connect-for-snmp/templates/redis/redis-config.yaml b/rendered/manifests/tests_only_polling/splunk-connect-for-snmp/templates/redis/redis-config.yaml new file mode 100644 index 000000000..22da33840 --- /dev/null +++ b/rendered/manifests/tests_only_polling/splunk-connect-for-snmp/templates/redis/redis-config.yaml @@ -0,0 +1,33 @@ +--- +# Source: splunk-connect-for-snmp/templates/redis/redis-config.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: release-name-redis-config + namespace: default + labels: + app: release-name-redis +data: + redis.conf: | + # Data directory + dir /data + + # Persistence - RDB + save 900 1 + save 300 10 + save 60 10000 + + # Persistence - AOF + appendonly yes + appendfsync everysec + + # Logging + loglevel notice + + # Memory + maxmemory-policy noeviction + + # Network + bind 0.0.0.0 + protected-mode no + port 6379 diff --git a/rendered/manifests/tests_only_polling/splunk-connect-for-snmp/templates/redis/redis-standalone-service.yaml b/rendered/manifests/tests_only_polling/splunk-connect-for-snmp/templates/redis/redis-standalone-service.yaml new file mode 100644 index 000000000..5b3445437 --- /dev/null +++ b/rendered/manifests/tests_only_polling/splunk-connect-for-snmp/templates/redis/redis-standalone-service.yaml @@ -0,0 +1,15 @@ +--- +# Source: splunk-connect-for-snmp/templates/redis/redis-standalone-service.yaml +apiVersion: v1 +kind: Service +metadata: + name: release-name-redis + namespace: default +spec: + type: ClusterIP + ports: + - port: 6379 + targetPort: 6379 + name: redis + selector: + app: release-name-redis diff --git a/rendered/manifests/tests_only_polling/splunk-connect-for-snmp/templates/redis/redis-standalone-statefulset.yaml b/rendered/manifests/tests_only_polling/splunk-connect-for-snmp/templates/redis/redis-standalone-statefulset.yaml new file mode 100644 index 000000000..b38e4ba21 --- /dev/null +++ b/rendered/manifests/tests_only_polling/splunk-connect-for-snmp/templates/redis/redis-standalone-statefulset.yaml @@ -0,0 +1,112 @@ +--- +# Source: splunk-connect-for-snmp/templates/redis/redis-standalone-statefulset.yaml +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: release-name-redis + namespace: default + labels: + app: release-name-redis +spec: + serviceName: release-name-redis + replicas: 1 + selector: + matchLabels: + app: release-name-redis + template: + metadata: + labels: + app: release-name-redis + annotations: + checksum/config: e82c09fa615350d9c147a0884485f953308babd6b8842d0cbe695ed5595eb530 + spec: + securityContext: + runAsUser: 999 + fsGroup: 999 + initContainers: + - name: fix-permissions + image: redis:8.2.2 + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - | + echo "=== Redis Init: Fixing Permissions ===" + echo "Current ownership:" + ls -ln /data + echo "" + echo "Fixing ownership to 999:999..." + chown -R 999:999 /data + chmod -R 755 /data + echo "" + echo "New ownership:" + ls -ln /data + echo "=== Permissions Fixed ===" + volumeMounts: + - name: redis-data + mountPath: /data + securityContext: + runAsUser: 0 # Must run as root to chown + containers: + - name: redis + image: redis:8.2.2 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 6379 + name: redis + command: + - sh + - -c + args: + - | + # Copy config to writable location + cp /etc/redis/redis.conf /tmp/redis.conf + + # Start Redis + exec redis-server /tmp/redis.conf + volumeMounts: + - name: redis-data + mountPath: /data + - name: redis-config + mountPath: /etc/redis + resources: + limits: + cpu: 500m + memory: 512Mi + requests: + cpu: 250m + memory: 256Mi + livenessProbe: + exec: + command: + - sh + - -c + - | + redis-cli ping + initialDelaySeconds: 30 + periodSeconds: 10 + readinessProbe: + exec: + command: + - sh + - -c + - | + redis-cli ping + initialDelaySeconds: 5 + periodSeconds: 5 + # Storage enabled but no existing PVC - use volumeClaimTemplates below + volumes: + - name: redis-config + configMap: + name: release-name-redis-config + # No existing PVC found, create new one via volumeClaimTemplates + volumeClaimTemplates: + - metadata: + name: redis-data + spec: + accessModes: + - ReadWriteOnce + storageClassName: microk8s-hostpath + resources: + requests: + storage: 5Gi diff --git a/rendered/manifests/tests_only_polling/splunk-connect-for-snmp/templates/scheduler/deployment.yaml b/rendered/manifests/tests_only_polling/splunk-connect-for-snmp/templates/scheduler/deployment.yaml index 40c223461..785864c76 100644 --- a/rendered/manifests/tests_only_polling/splunk-connect-for-snmp/templates/scheduler/deployment.yaml +++ b/rendered/manifests/tests_only_polling/splunk-connect-for-snmp/templates/scheduler/deployment.yaml @@ -44,10 +44,14 @@ spec: env: - name: CONFIG_PATH value: /app/config/config.yaml - - name: REDIS_URL - value: redis://release-name-redis-master:6379/1 - - name: CELERY_BROKER_URL - value: redis://release-name-redis-master:6379/0 + - name: REDIS_HOST + value: release-name-redis + - name: REDIS_PORT + value: "6379" + - name: REDIS_DB + value: "1" + - name: CELERY_DB + value: "0" - name: MONGO_URI value: mongodb://release-name-mongodb:27017 - name: MIB_SOURCES diff --git a/rendered/manifests/tests_only_polling/splunk-connect-for-snmp/templates/worker/poller/deployment.yaml b/rendered/manifests/tests_only_polling/splunk-connect-for-snmp/templates/worker/poller/deployment.yaml index 01dfdcb78..d5f4381ed 100644 --- a/rendered/manifests/tests_only_polling/splunk-connect-for-snmp/templates/worker/poller/deployment.yaml +++ b/rendered/manifests/tests_only_polling/splunk-connect-for-snmp/templates/worker/poller/deployment.yaml @@ -44,12 +44,16 @@ spec: env: - name: CONFIG_PATH value: /app/config/config.yaml - - name: REDIS_URL - value: redis://release-name-redis-master:6379/1 + - name: REDIS_HOST + value: release-name-redis + - name: REDIS_PORT + value: "6379" + - name: REDIS_DB + value: "1" + - name: CELERY_DB + value: "0" - name: SC4SNMP_VERSION value: CURRENT-VERSION - - name: CELERY_BROKER_URL - value: redis://release-name-redis-master:6379/0 - name: MONGO_URI value: mongodb://release-name-mongodb:27017 - name: WALK_RETRY_MAX_INTERVAL @@ -66,6 +70,8 @@ spec: value: "true" - name: UDP_CONNECTION_TIMEOUT value: "3" + - name: UDP_CONNECTION_RETRIES + value: "5" - name: MAX_OID_TO_PROCESS value: "70" - name: MAX_REPETITIONS diff --git a/rendered/manifests/tests_only_polling/splunk-connect-for-snmp/templates/worker/sender/deployment.yaml b/rendered/manifests/tests_only_polling/splunk-connect-for-snmp/templates/worker/sender/deployment.yaml index fe08ce4ed..7e2ec223b 100644 --- a/rendered/manifests/tests_only_polling/splunk-connect-for-snmp/templates/worker/sender/deployment.yaml +++ b/rendered/manifests/tests_only_polling/splunk-connect-for-snmp/templates/worker/sender/deployment.yaml @@ -44,12 +44,16 @@ spec: env: - name: CONFIG_PATH value: /app/config/config.yaml - - name: REDIS_URL - value: redis://release-name-redis-master:6379/1 + - name: REDIS_HOST + value: release-name-redis + - name: REDIS_PORT + value: "6379" + - name: REDIS_DB + value: "1" + - name: CELERY_DB + value: "0" - name: SC4SNMP_VERSION value: CURRENT-VERSION - - name: CELERY_BROKER_URL - value: redis://release-name-redis-master:6379/0 - name: MONGO_URI value: mongodb://release-name-mongodb:27017 - name: WALK_RETRY_MAX_INTERVAL @@ -66,6 +70,8 @@ spec: value: "true" - name: UDP_CONNECTION_TIMEOUT value: "3" + - name: UDP_CONNECTION_RETRIES + value: "5" - name: MAX_OID_TO_PROCESS value: "70" - name: MAX_REPETITIONS diff --git a/rendered/manifests/tests_only_traps/splunk-connect-for-snmp/templates/common/discovery-config.yaml b/rendered/manifests/tests_only_traps/splunk-connect-for-snmp/templates/common/discovery-config.yaml new file mode 100644 index 000000000..8f5acde96 --- /dev/null +++ b/rendered/manifests/tests_only_traps/splunk-connect-for-snmp/templates/common/discovery-config.yaml @@ -0,0 +1,12 @@ +--- +# Source: splunk-connect-for-snmp/templates/common/discovery-config.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: splunk-connect-for-snmp-discovery-config +data: + discovery-config.yaml: | + enabled: false + ipv6Enabled: false + logLevel: INFO + usernameSecrets: [] diff --git a/rendered/manifests/tests_only_traps/splunk-connect-for-snmp/templates/redis/redis-config.yaml b/rendered/manifests/tests_only_traps/splunk-connect-for-snmp/templates/redis/redis-config.yaml new file mode 100644 index 000000000..22da33840 --- /dev/null +++ b/rendered/manifests/tests_only_traps/splunk-connect-for-snmp/templates/redis/redis-config.yaml @@ -0,0 +1,33 @@ +--- +# Source: splunk-connect-for-snmp/templates/redis/redis-config.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: release-name-redis-config + namespace: default + labels: + app: release-name-redis +data: + redis.conf: | + # Data directory + dir /data + + # Persistence - RDB + save 900 1 + save 300 10 + save 60 10000 + + # Persistence - AOF + appendonly yes + appendfsync everysec + + # Logging + loglevel notice + + # Memory + maxmemory-policy noeviction + + # Network + bind 0.0.0.0 + protected-mode no + port 6379 diff --git a/rendered/manifests/tests_only_traps/splunk-connect-for-snmp/templates/redis/redis-standalone-service.yaml b/rendered/manifests/tests_only_traps/splunk-connect-for-snmp/templates/redis/redis-standalone-service.yaml new file mode 100644 index 000000000..5b3445437 --- /dev/null +++ b/rendered/manifests/tests_only_traps/splunk-connect-for-snmp/templates/redis/redis-standalone-service.yaml @@ -0,0 +1,15 @@ +--- +# Source: splunk-connect-for-snmp/templates/redis/redis-standalone-service.yaml +apiVersion: v1 +kind: Service +metadata: + name: release-name-redis + namespace: default +spec: + type: ClusterIP + ports: + - port: 6379 + targetPort: 6379 + name: redis + selector: + app: release-name-redis diff --git a/rendered/manifests/tests_only_traps/splunk-connect-for-snmp/templates/redis/redis-standalone-statefulset.yaml b/rendered/manifests/tests_only_traps/splunk-connect-for-snmp/templates/redis/redis-standalone-statefulset.yaml new file mode 100644 index 000000000..b38e4ba21 --- /dev/null +++ b/rendered/manifests/tests_only_traps/splunk-connect-for-snmp/templates/redis/redis-standalone-statefulset.yaml @@ -0,0 +1,112 @@ +--- +# Source: splunk-connect-for-snmp/templates/redis/redis-standalone-statefulset.yaml +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: release-name-redis + namespace: default + labels: + app: release-name-redis +spec: + serviceName: release-name-redis + replicas: 1 + selector: + matchLabels: + app: release-name-redis + template: + metadata: + labels: + app: release-name-redis + annotations: + checksum/config: e82c09fa615350d9c147a0884485f953308babd6b8842d0cbe695ed5595eb530 + spec: + securityContext: + runAsUser: 999 + fsGroup: 999 + initContainers: + - name: fix-permissions + image: redis:8.2.2 + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - | + echo "=== Redis Init: Fixing Permissions ===" + echo "Current ownership:" + ls -ln /data + echo "" + echo "Fixing ownership to 999:999..." + chown -R 999:999 /data + chmod -R 755 /data + echo "" + echo "New ownership:" + ls -ln /data + echo "=== Permissions Fixed ===" + volumeMounts: + - name: redis-data + mountPath: /data + securityContext: + runAsUser: 0 # Must run as root to chown + containers: + - name: redis + image: redis:8.2.2 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 6379 + name: redis + command: + - sh + - -c + args: + - | + # Copy config to writable location + cp /etc/redis/redis.conf /tmp/redis.conf + + # Start Redis + exec redis-server /tmp/redis.conf + volumeMounts: + - name: redis-data + mountPath: /data + - name: redis-config + mountPath: /etc/redis + resources: + limits: + cpu: 500m + memory: 512Mi + requests: + cpu: 250m + memory: 256Mi + livenessProbe: + exec: + command: + - sh + - -c + - | + redis-cli ping + initialDelaySeconds: 30 + periodSeconds: 10 + readinessProbe: + exec: + command: + - sh + - -c + - | + redis-cli ping + initialDelaySeconds: 5 + periodSeconds: 5 + # Storage enabled but no existing PVC - use volumeClaimTemplates below + volumes: + - name: redis-config + configMap: + name: release-name-redis-config + # No existing PVC found, create new one via volumeClaimTemplates + volumeClaimTemplates: + - metadata: + name: redis-data + spec: + accessModes: + - ReadWriteOnce + storageClassName: microk8s-hostpath + resources: + requests: + storage: 5Gi diff --git a/rendered/manifests/tests_only_traps/splunk-connect-for-snmp/templates/traps/deployment.yaml b/rendered/manifests/tests_only_traps/splunk-connect-for-snmp/templates/traps/deployment.yaml index 65c9b834f..db1be6303 100644 --- a/rendered/manifests/tests_only_traps/splunk-connect-for-snmp/templates/traps/deployment.yaml +++ b/rendered/manifests/tests_only_traps/splunk-connect-for-snmp/templates/traps/deployment.yaml @@ -44,10 +44,16 @@ spec: env: - name: CONFIG_PATH value: /app/config/config.yaml - - name: CELERY_BROKER_URL - value: redis://release-name-redis-master:6379/0 - name: MONGO_URI value: mongodb://release-name-mongodb:27017 + - name: REDIS_HOST + value: release-name-redis + - name: REDIS_PORT + value: "6379" + - name: REDIS_DB + value: "1" + - name: CELERY_DB + value: "0" - name: MIB_SOURCES value: "http://release-name-mibserver/asn1/@mib@" - name: MIB_INDEX diff --git a/rendered/manifests/tests_only_traps/splunk-connect-for-snmp/templates/worker/sender/deployment.yaml b/rendered/manifests/tests_only_traps/splunk-connect-for-snmp/templates/worker/sender/deployment.yaml index fe08ce4ed..7e2ec223b 100644 --- a/rendered/manifests/tests_only_traps/splunk-connect-for-snmp/templates/worker/sender/deployment.yaml +++ b/rendered/manifests/tests_only_traps/splunk-connect-for-snmp/templates/worker/sender/deployment.yaml @@ -44,12 +44,16 @@ spec: env: - name: CONFIG_PATH value: /app/config/config.yaml - - name: REDIS_URL - value: redis://release-name-redis-master:6379/1 + - name: REDIS_HOST + value: release-name-redis + - name: REDIS_PORT + value: "6379" + - name: REDIS_DB + value: "1" + - name: CELERY_DB + value: "0" - name: SC4SNMP_VERSION value: CURRENT-VERSION - - name: CELERY_BROKER_URL - value: redis://release-name-redis-master:6379/0 - name: MONGO_URI value: mongodb://release-name-mongodb:27017 - name: WALK_RETRY_MAX_INTERVAL @@ -66,6 +70,8 @@ spec: value: "true" - name: UDP_CONNECTION_TIMEOUT value: "3" + - name: UDP_CONNECTION_RETRIES + value: "5" - name: MAX_OID_TO_PROCESS value: "70" - name: MAX_REPETITIONS diff --git a/rendered/manifests/tests_only_traps/splunk-connect-for-snmp/templates/worker/trap/deployment.yaml b/rendered/manifests/tests_only_traps/splunk-connect-for-snmp/templates/worker/trap/deployment.yaml index 82675bad1..d506f7a42 100644 --- a/rendered/manifests/tests_only_traps/splunk-connect-for-snmp/templates/worker/trap/deployment.yaml +++ b/rendered/manifests/tests_only_traps/splunk-connect-for-snmp/templates/worker/trap/deployment.yaml @@ -44,12 +44,16 @@ spec: env: - name: CONFIG_PATH value: /app/config/config.yaml - - name: REDIS_URL - value: redis://release-name-redis-master:6379/1 + - name: REDIS_HOST + value: release-name-redis + - name: REDIS_PORT + value: "6379" + - name: REDIS_DB + value: "1" + - name: CELERY_DB + value: "0" - name: SC4SNMP_VERSION value: CURRENT-VERSION - - name: CELERY_BROKER_URL - value: redis://release-name-redis-master:6379/0 - name: MONGO_URI value: mongodb://release-name-mongodb:27017 - name: WALK_RETRY_MAX_INTERVAL @@ -66,6 +70,8 @@ spec: value: "true" - name: UDP_CONNECTION_TIMEOUT value: "3" + - name: UDP_CONNECTION_RETRIES + value: "5" - name: MAX_OID_TO_PROCESS value: "70" - name: MAX_REPETITIONS diff --git a/rendered/manifests/tests_probes_enabled/splunk-connect-for-snmp/templates/common/discovery-config.yaml b/rendered/manifests/tests_probes_enabled/splunk-connect-for-snmp/templates/common/discovery-config.yaml new file mode 100644 index 000000000..8f5acde96 --- /dev/null +++ b/rendered/manifests/tests_probes_enabled/splunk-connect-for-snmp/templates/common/discovery-config.yaml @@ -0,0 +1,12 @@ +--- +# Source: splunk-connect-for-snmp/templates/common/discovery-config.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: splunk-connect-for-snmp-discovery-config +data: + discovery-config.yaml: | + enabled: false + ipv6Enabled: false + logLevel: INFO + usernameSecrets: [] diff --git a/rendered/manifests/tests_probes_enabled/splunk-connect-for-snmp/templates/inventory/job.yaml b/rendered/manifests/tests_probes_enabled/splunk-connect-for-snmp/templates/inventory/job.yaml index 3ea664cc0..2708b9ffd 100644 --- a/rendered/manifests/tests_probes_enabled/splunk-connect-for-snmp/templates/inventory/job.yaml +++ b/rendered/manifests/tests_probes_enabled/splunk-connect-for-snmp/templates/inventory/job.yaml @@ -28,12 +28,16 @@ spec: env: - name: CONFIG_PATH value: /app/config/config.yaml - - name: REDIS_URL - value: redis://release-name-redis-master:6379/1 + - name: REDIS_HOST + value: release-name-redis + - name: REDIS_PORT + value: "6379" + - name: REDIS_DB + value: "1" + - name: CELERY_DB + value: "0" - name: INVENTORY_PATH value: /app/inventory/inventory.csv - - name: CELERY_BROKER_URL - value: redis://release-name-redis-master:6379/0 - name: MONGO_URI value: mongodb://release-name-mongodb:27017 - name: MIB_SOURCES diff --git a/rendered/manifests/tests_probes_enabled/splunk-connect-for-snmp/templates/redis/redis-config.yaml b/rendered/manifests/tests_probes_enabled/splunk-connect-for-snmp/templates/redis/redis-config.yaml new file mode 100644 index 000000000..22da33840 --- /dev/null +++ b/rendered/manifests/tests_probes_enabled/splunk-connect-for-snmp/templates/redis/redis-config.yaml @@ -0,0 +1,33 @@ +--- +# Source: splunk-connect-for-snmp/templates/redis/redis-config.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: release-name-redis-config + namespace: default + labels: + app: release-name-redis +data: + redis.conf: | + # Data directory + dir /data + + # Persistence - RDB + save 900 1 + save 300 10 + save 60 10000 + + # Persistence - AOF + appendonly yes + appendfsync everysec + + # Logging + loglevel notice + + # Memory + maxmemory-policy noeviction + + # Network + bind 0.0.0.0 + protected-mode no + port 6379 diff --git a/rendered/manifests/tests_probes_enabled/splunk-connect-for-snmp/templates/redis/redis-standalone-service.yaml b/rendered/manifests/tests_probes_enabled/splunk-connect-for-snmp/templates/redis/redis-standalone-service.yaml new file mode 100644 index 000000000..5b3445437 --- /dev/null +++ b/rendered/manifests/tests_probes_enabled/splunk-connect-for-snmp/templates/redis/redis-standalone-service.yaml @@ -0,0 +1,15 @@ +--- +# Source: splunk-connect-for-snmp/templates/redis/redis-standalone-service.yaml +apiVersion: v1 +kind: Service +metadata: + name: release-name-redis + namespace: default +spec: + type: ClusterIP + ports: + - port: 6379 + targetPort: 6379 + name: redis + selector: + app: release-name-redis diff --git a/rendered/manifests/tests_probes_enabled/splunk-connect-for-snmp/templates/redis/redis-standalone-statefulset.yaml b/rendered/manifests/tests_probes_enabled/splunk-connect-for-snmp/templates/redis/redis-standalone-statefulset.yaml new file mode 100644 index 000000000..b38e4ba21 --- /dev/null +++ b/rendered/manifests/tests_probes_enabled/splunk-connect-for-snmp/templates/redis/redis-standalone-statefulset.yaml @@ -0,0 +1,112 @@ +--- +# Source: splunk-connect-for-snmp/templates/redis/redis-standalone-statefulset.yaml +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: release-name-redis + namespace: default + labels: + app: release-name-redis +spec: + serviceName: release-name-redis + replicas: 1 + selector: + matchLabels: + app: release-name-redis + template: + metadata: + labels: + app: release-name-redis + annotations: + checksum/config: e82c09fa615350d9c147a0884485f953308babd6b8842d0cbe695ed5595eb530 + spec: + securityContext: + runAsUser: 999 + fsGroup: 999 + initContainers: + - name: fix-permissions + image: redis:8.2.2 + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - | + echo "=== Redis Init: Fixing Permissions ===" + echo "Current ownership:" + ls -ln /data + echo "" + echo "Fixing ownership to 999:999..." + chown -R 999:999 /data + chmod -R 755 /data + echo "" + echo "New ownership:" + ls -ln /data + echo "=== Permissions Fixed ===" + volumeMounts: + - name: redis-data + mountPath: /data + securityContext: + runAsUser: 0 # Must run as root to chown + containers: + - name: redis + image: redis:8.2.2 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 6379 + name: redis + command: + - sh + - -c + args: + - | + # Copy config to writable location + cp /etc/redis/redis.conf /tmp/redis.conf + + # Start Redis + exec redis-server /tmp/redis.conf + volumeMounts: + - name: redis-data + mountPath: /data + - name: redis-config + mountPath: /etc/redis + resources: + limits: + cpu: 500m + memory: 512Mi + requests: + cpu: 250m + memory: 256Mi + livenessProbe: + exec: + command: + - sh + - -c + - | + redis-cli ping + initialDelaySeconds: 30 + periodSeconds: 10 + readinessProbe: + exec: + command: + - sh + - -c + - | + redis-cli ping + initialDelaySeconds: 5 + periodSeconds: 5 + # Storage enabled but no existing PVC - use volumeClaimTemplates below + volumes: + - name: redis-config + configMap: + name: release-name-redis-config + # No existing PVC found, create new one via volumeClaimTemplates + volumeClaimTemplates: + - metadata: + name: redis-data + spec: + accessModes: + - ReadWriteOnce + storageClassName: microk8s-hostpath + resources: + requests: + storage: 5Gi diff --git a/rendered/manifests/tests_probes_enabled/splunk-connect-for-snmp/templates/scheduler/deployment.yaml b/rendered/manifests/tests_probes_enabled/splunk-connect-for-snmp/templates/scheduler/deployment.yaml index 40c223461..785864c76 100644 --- a/rendered/manifests/tests_probes_enabled/splunk-connect-for-snmp/templates/scheduler/deployment.yaml +++ b/rendered/manifests/tests_probes_enabled/splunk-connect-for-snmp/templates/scheduler/deployment.yaml @@ -44,10 +44,14 @@ spec: env: - name: CONFIG_PATH value: /app/config/config.yaml - - name: REDIS_URL - value: redis://release-name-redis-master:6379/1 - - name: CELERY_BROKER_URL - value: redis://release-name-redis-master:6379/0 + - name: REDIS_HOST + value: release-name-redis + - name: REDIS_PORT + value: "6379" + - name: REDIS_DB + value: "1" + - name: CELERY_DB + value: "0" - name: MONGO_URI value: mongodb://release-name-mongodb:27017 - name: MIB_SOURCES diff --git a/rendered/manifests/tests_probes_enabled/splunk-connect-for-snmp/templates/traps/deployment.yaml b/rendered/manifests/tests_probes_enabled/splunk-connect-for-snmp/templates/traps/deployment.yaml index 65c9b834f..db1be6303 100644 --- a/rendered/manifests/tests_probes_enabled/splunk-connect-for-snmp/templates/traps/deployment.yaml +++ b/rendered/manifests/tests_probes_enabled/splunk-connect-for-snmp/templates/traps/deployment.yaml @@ -44,10 +44,16 @@ spec: env: - name: CONFIG_PATH value: /app/config/config.yaml - - name: CELERY_BROKER_URL - value: redis://release-name-redis-master:6379/0 - name: MONGO_URI value: mongodb://release-name-mongodb:27017 + - name: REDIS_HOST + value: release-name-redis + - name: REDIS_PORT + value: "6379" + - name: REDIS_DB + value: "1" + - name: CELERY_DB + value: "0" - name: MIB_SOURCES value: "http://release-name-mibserver/asn1/@mib@" - name: MIB_INDEX diff --git a/rendered/manifests/tests_probes_enabled/splunk-connect-for-snmp/templates/worker/poller/deployment.yaml b/rendered/manifests/tests_probes_enabled/splunk-connect-for-snmp/templates/worker/poller/deployment.yaml index 61b4fa2de..402eadfb5 100644 --- a/rendered/manifests/tests_probes_enabled/splunk-connect-for-snmp/templates/worker/poller/deployment.yaml +++ b/rendered/manifests/tests_probes_enabled/splunk-connect-for-snmp/templates/worker/poller/deployment.yaml @@ -44,12 +44,16 @@ spec: env: - name: CONFIG_PATH value: /app/config/config.yaml - - name: REDIS_URL - value: redis://release-name-redis-master:6379/1 + - name: REDIS_HOST + value: release-name-redis + - name: REDIS_PORT + value: "6379" + - name: REDIS_DB + value: "1" + - name: CELERY_DB + value: "0" - name: SC4SNMP_VERSION value: CURRENT-VERSION - - name: CELERY_BROKER_URL - value: redis://release-name-redis-master:6379/0 - name: MONGO_URI value: mongodb://release-name-mongodb:27017 - name: WALK_RETRY_MAX_INTERVAL @@ -66,6 +70,8 @@ spec: value: "true" - name: UDP_CONNECTION_TIMEOUT value: "3" + - name: UDP_CONNECTION_RETRIES + value: "5" - name: MAX_OID_TO_PROCESS value: "70" - name: MAX_REPETITIONS diff --git a/rendered/manifests/tests_probes_enabled/splunk-connect-for-snmp/templates/worker/sender/deployment.yaml b/rendered/manifests/tests_probes_enabled/splunk-connect-for-snmp/templates/worker/sender/deployment.yaml index 2235659a2..498ce4808 100644 --- a/rendered/manifests/tests_probes_enabled/splunk-connect-for-snmp/templates/worker/sender/deployment.yaml +++ b/rendered/manifests/tests_probes_enabled/splunk-connect-for-snmp/templates/worker/sender/deployment.yaml @@ -44,12 +44,16 @@ spec: env: - name: CONFIG_PATH value: /app/config/config.yaml - - name: REDIS_URL - value: redis://release-name-redis-master:6379/1 + - name: REDIS_HOST + value: release-name-redis + - name: REDIS_PORT + value: "6379" + - name: REDIS_DB + value: "1" + - name: CELERY_DB + value: "0" - name: SC4SNMP_VERSION value: CURRENT-VERSION - - name: CELERY_BROKER_URL - value: redis://release-name-redis-master:6379/0 - name: MONGO_URI value: mongodb://release-name-mongodb:27017 - name: WALK_RETRY_MAX_INTERVAL @@ -66,6 +70,8 @@ spec: value: "true" - name: UDP_CONNECTION_TIMEOUT value: "3" + - name: UDP_CONNECTION_RETRIES + value: "5" - name: MAX_OID_TO_PROCESS value: "70" - name: MAX_REPETITIONS diff --git a/rendered/manifests/tests_probes_enabled/splunk-connect-for-snmp/templates/worker/trap/deployment.yaml b/rendered/manifests/tests_probes_enabled/splunk-connect-for-snmp/templates/worker/trap/deployment.yaml index b699b77f8..891c69896 100644 --- a/rendered/manifests/tests_probes_enabled/splunk-connect-for-snmp/templates/worker/trap/deployment.yaml +++ b/rendered/manifests/tests_probes_enabled/splunk-connect-for-snmp/templates/worker/trap/deployment.yaml @@ -44,12 +44,16 @@ spec: env: - name: CONFIG_PATH value: /app/config/config.yaml - - name: REDIS_URL - value: redis://release-name-redis-master:6379/1 + - name: REDIS_HOST + value: release-name-redis + - name: REDIS_PORT + value: "6379" + - name: REDIS_DB + value: "1" + - name: CELERY_DB + value: "0" - name: SC4SNMP_VERSION value: CURRENT-VERSION - - name: CELERY_BROKER_URL - value: redis://release-name-redis-master:6379/0 - name: MONGO_URI value: mongodb://release-name-mongodb:27017 - name: WALK_RETRY_MAX_INTERVAL @@ -66,6 +70,8 @@ spec: value: "true" - name: UDP_CONNECTION_TIMEOUT value: "3" + - name: UDP_CONNECTION_RETRIES + value: "5" - name: MAX_OID_TO_PROCESS value: "70" - name: MAX_REPETITIONS diff --git a/rendered/manifests/tests_traps_nodeport/splunk-connect-for-snmp/templates/common/discovery-config.yaml b/rendered/manifests/tests_traps_nodeport/splunk-connect-for-snmp/templates/common/discovery-config.yaml new file mode 100644 index 000000000..8f5acde96 --- /dev/null +++ b/rendered/manifests/tests_traps_nodeport/splunk-connect-for-snmp/templates/common/discovery-config.yaml @@ -0,0 +1,12 @@ +--- +# Source: splunk-connect-for-snmp/templates/common/discovery-config.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: splunk-connect-for-snmp-discovery-config +data: + discovery-config.yaml: | + enabled: false + ipv6Enabled: false + logLevel: INFO + usernameSecrets: [] diff --git a/rendered/manifests/tests_traps_nodeport/splunk-connect-for-snmp/templates/redis/redis-config.yaml b/rendered/manifests/tests_traps_nodeport/splunk-connect-for-snmp/templates/redis/redis-config.yaml new file mode 100644 index 000000000..22da33840 --- /dev/null +++ b/rendered/manifests/tests_traps_nodeport/splunk-connect-for-snmp/templates/redis/redis-config.yaml @@ -0,0 +1,33 @@ +--- +# Source: splunk-connect-for-snmp/templates/redis/redis-config.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: release-name-redis-config + namespace: default + labels: + app: release-name-redis +data: + redis.conf: | + # Data directory + dir /data + + # Persistence - RDB + save 900 1 + save 300 10 + save 60 10000 + + # Persistence - AOF + appendonly yes + appendfsync everysec + + # Logging + loglevel notice + + # Memory + maxmemory-policy noeviction + + # Network + bind 0.0.0.0 + protected-mode no + port 6379 diff --git a/rendered/manifests/tests_traps_nodeport/splunk-connect-for-snmp/templates/redis/redis-standalone-service.yaml b/rendered/manifests/tests_traps_nodeport/splunk-connect-for-snmp/templates/redis/redis-standalone-service.yaml new file mode 100644 index 000000000..5b3445437 --- /dev/null +++ b/rendered/manifests/tests_traps_nodeport/splunk-connect-for-snmp/templates/redis/redis-standalone-service.yaml @@ -0,0 +1,15 @@ +--- +# Source: splunk-connect-for-snmp/templates/redis/redis-standalone-service.yaml +apiVersion: v1 +kind: Service +metadata: + name: release-name-redis + namespace: default +spec: + type: ClusterIP + ports: + - port: 6379 + targetPort: 6379 + name: redis + selector: + app: release-name-redis diff --git a/rendered/manifests/tests_traps_nodeport/splunk-connect-for-snmp/templates/redis/redis-standalone-statefulset.yaml b/rendered/manifests/tests_traps_nodeport/splunk-connect-for-snmp/templates/redis/redis-standalone-statefulset.yaml new file mode 100644 index 000000000..b38e4ba21 --- /dev/null +++ b/rendered/manifests/tests_traps_nodeport/splunk-connect-for-snmp/templates/redis/redis-standalone-statefulset.yaml @@ -0,0 +1,112 @@ +--- +# Source: splunk-connect-for-snmp/templates/redis/redis-standalone-statefulset.yaml +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: release-name-redis + namespace: default + labels: + app: release-name-redis +spec: + serviceName: release-name-redis + replicas: 1 + selector: + matchLabels: + app: release-name-redis + template: + metadata: + labels: + app: release-name-redis + annotations: + checksum/config: e82c09fa615350d9c147a0884485f953308babd6b8842d0cbe695ed5595eb530 + spec: + securityContext: + runAsUser: 999 + fsGroup: 999 + initContainers: + - name: fix-permissions + image: redis:8.2.2 + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - | + echo "=== Redis Init: Fixing Permissions ===" + echo "Current ownership:" + ls -ln /data + echo "" + echo "Fixing ownership to 999:999..." + chown -R 999:999 /data + chmod -R 755 /data + echo "" + echo "New ownership:" + ls -ln /data + echo "=== Permissions Fixed ===" + volumeMounts: + - name: redis-data + mountPath: /data + securityContext: + runAsUser: 0 # Must run as root to chown + containers: + - name: redis + image: redis:8.2.2 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 6379 + name: redis + command: + - sh + - -c + args: + - | + # Copy config to writable location + cp /etc/redis/redis.conf /tmp/redis.conf + + # Start Redis + exec redis-server /tmp/redis.conf + volumeMounts: + - name: redis-data + mountPath: /data + - name: redis-config + mountPath: /etc/redis + resources: + limits: + cpu: 500m + memory: 512Mi + requests: + cpu: 250m + memory: 256Mi + livenessProbe: + exec: + command: + - sh + - -c + - | + redis-cli ping + initialDelaySeconds: 30 + periodSeconds: 10 + readinessProbe: + exec: + command: + - sh + - -c + - | + redis-cli ping + initialDelaySeconds: 5 + periodSeconds: 5 + # Storage enabled but no existing PVC - use volumeClaimTemplates below + volumes: + - name: redis-config + configMap: + name: release-name-redis-config + # No existing PVC found, create new one via volumeClaimTemplates + volumeClaimTemplates: + - metadata: + name: redis-data + spec: + accessModes: + - ReadWriteOnce + storageClassName: microk8s-hostpath + resources: + requests: + storage: 5Gi diff --git a/rendered/manifests/tests_traps_nodeport/splunk-connect-for-snmp/templates/traps/deployment.yaml b/rendered/manifests/tests_traps_nodeport/splunk-connect-for-snmp/templates/traps/deployment.yaml index 65c9b834f..db1be6303 100644 --- a/rendered/manifests/tests_traps_nodeport/splunk-connect-for-snmp/templates/traps/deployment.yaml +++ b/rendered/manifests/tests_traps_nodeport/splunk-connect-for-snmp/templates/traps/deployment.yaml @@ -44,10 +44,16 @@ spec: env: - name: CONFIG_PATH value: /app/config/config.yaml - - name: CELERY_BROKER_URL - value: redis://release-name-redis-master:6379/0 - name: MONGO_URI value: mongodb://release-name-mongodb:27017 + - name: REDIS_HOST + value: release-name-redis + - name: REDIS_PORT + value: "6379" + - name: REDIS_DB + value: "1" + - name: CELERY_DB + value: "0" - name: MIB_SOURCES value: "http://release-name-mibserver/asn1/@mib@" - name: MIB_INDEX diff --git a/rendered/manifests/tests_traps_nodeport/splunk-connect-for-snmp/templates/worker/sender/deployment.yaml b/rendered/manifests/tests_traps_nodeport/splunk-connect-for-snmp/templates/worker/sender/deployment.yaml index fe08ce4ed..7e2ec223b 100644 --- a/rendered/manifests/tests_traps_nodeport/splunk-connect-for-snmp/templates/worker/sender/deployment.yaml +++ b/rendered/manifests/tests_traps_nodeport/splunk-connect-for-snmp/templates/worker/sender/deployment.yaml @@ -44,12 +44,16 @@ spec: env: - name: CONFIG_PATH value: /app/config/config.yaml - - name: REDIS_URL - value: redis://release-name-redis-master:6379/1 + - name: REDIS_HOST + value: release-name-redis + - name: REDIS_PORT + value: "6379" + - name: REDIS_DB + value: "1" + - name: CELERY_DB + value: "0" - name: SC4SNMP_VERSION value: CURRENT-VERSION - - name: CELERY_BROKER_URL - value: redis://release-name-redis-master:6379/0 - name: MONGO_URI value: mongodb://release-name-mongodb:27017 - name: WALK_RETRY_MAX_INTERVAL @@ -66,6 +70,8 @@ spec: value: "true" - name: UDP_CONNECTION_TIMEOUT value: "3" + - name: UDP_CONNECTION_RETRIES + value: "5" - name: MAX_OID_TO_PROCESS value: "70" - name: MAX_REPETITIONS diff --git a/rendered/manifests/tests_traps_nodeport/splunk-connect-for-snmp/templates/worker/trap/deployment.yaml b/rendered/manifests/tests_traps_nodeport/splunk-connect-for-snmp/templates/worker/trap/deployment.yaml index 82675bad1..d506f7a42 100644 --- a/rendered/manifests/tests_traps_nodeport/splunk-connect-for-snmp/templates/worker/trap/deployment.yaml +++ b/rendered/manifests/tests_traps_nodeport/splunk-connect-for-snmp/templates/worker/trap/deployment.yaml @@ -44,12 +44,16 @@ spec: env: - name: CONFIG_PATH value: /app/config/config.yaml - - name: REDIS_URL - value: redis://release-name-redis-master:6379/1 + - name: REDIS_HOST + value: release-name-redis + - name: REDIS_PORT + value: "6379" + - name: REDIS_DB + value: "1" + - name: CELERY_DB + value: "0" - name: SC4SNMP_VERSION value: CURRENT-VERSION - - name: CELERY_BROKER_URL - value: redis://release-name-redis-master:6379/0 - name: MONGO_URI value: mongodb://release-name-mongodb:27017 - name: WALK_RETRY_MAX_INTERVAL @@ -66,6 +70,8 @@ spec: value: "true" - name: UDP_CONNECTION_TIMEOUT value: "3" + - name: UDP_CONNECTION_RETRIES + value: "5" - name: MAX_OID_TO_PROCESS value: "70" - name: MAX_REPETITIONS diff --git a/rendered/values.yaml b/rendered/values.yaml index b65c765af..c02fcb391 100644 --- a/rendered/values.yaml +++ b/rendered/values.yaml @@ -36,4 +36,24 @@ scheduler: poller: inventory: | address,port,version,community,secret,security_engine,walk_interval,profiles,smart_profiles,delete - 54.82.41.24,,2c,public,,,1800,IF_profile,false, \ No newline at end of file + 54.82.41.24,,2c,public,,,1800,IF_profile,false, + +discovery: + enabled: true + ipv6Enabled: true + discoveryPath: "/home/devuser/discovery" + autodiscovery: + discovery_version2c: + frequency: 21600 + delete_already_discovered: false + network_address: 54.82.41.24/28 + version: "2c" + community: "public" + port: 161 + device_rules: + - name: "Linux servers" + patterns: "*linux*" + group: "linux_group" + - name: "Centos servers" + patterns: "*centos*" + group: "centos_group" diff --git a/rendered/values_only_discovery.yaml b/rendered/values_only_discovery.yaml new file mode 100644 index 000000000..89b2a278e --- /dev/null +++ b/rendered/values_only_discovery.yaml @@ -0,0 +1,34 @@ +discovery: + enabled: true + ipv6Enabled: true + discoveryPath: "/home/devuser/discovery-test2" + usernameSecrets: + - secret + + autodiscovery: + discovery_version2c: + frequency: 21600 + delete_already_discovered: false + network_address: 4.91.99.113/28 + version: "2c" + community: "public" + port: 161 + device_rules: + - name: "Linux servers" + patterns: "*linux*" + group: "linux_group" + - name: "Centos servers" + patterns: "*centos*" + group: "centos_group" + discovery_version3: + frequency: 21600 + delete_already_discovered: false + network_address: 4.91.99.113/28 + version: "3" + port: 161 + secret: secret + security_engine: "80001f8880e761866965756b6800000000" + device_rules: + - name: "Linux VM" + patterns: "*Linux*" + group: "linux_group" \ No newline at end of file diff --git a/splunk_connect_for_snmp/celery_config.py b/splunk_connect_for_snmp/celery_config.py index da2db670e..aadc73fdb 100644 --- a/splunk_connect_for_snmp/celery_config.py +++ b/splunk_connect_for_snmp/celery_config.py @@ -83,4 +83,5 @@ Queue("traps", exchange="traps"), Queue("poll", exchange="poll"), Queue("send", exchange="send"), + Queue("discovery", exchange="discovery"), ) diff --git a/splunk_connect_for_snmp/common/csv_record_manager.py b/splunk_connect_for_snmp/common/csv_record_manager.py new file mode 100644 index 000000000..55dee4728 --- /dev/null +++ b/splunk_connect_for_snmp/common/csv_record_manager.py @@ -0,0 +1,83 @@ +import csv +import os + +from celery.utils.log import get_task_logger + +logger = get_task_logger(__name__) + + +class CSVRecordManager: + def __init__(self, filename): + self.filename = filename + self.columns = [ + "key", + "subnet", + "ip", + "port", + "version", + "group", + "secret", + "community", + ] + + try: + if not os.path.isfile(filename): + with open(filename, mode="w", newline="") as f: + writer = csv.DictWriter(f, fieldnames=self.columns) + writer.writeheader() + self.rows = [] + else: + with open(filename, newline="") as f: + reader = csv.DictReader(f) + self.rows = list(reader) + except Exception as e: + logger.error(f"Error occurred while reading CSV file: {e}") + raise + + def _normalize_row(self, row: dict) -> dict: + """Strip whitespace and ensure all keys exist with empty string defaults.""" + return {k: str(v).strip() if v is not None else "" for k, v in row.items()} + + def _write_to_csv(self): + """Save current rows back to the CSV file.""" + try: + with open(self.filename, mode="w", newline="") as f: + writer = csv.DictWriter(f, fieldnames=self.columns) + writer.writeheader() + writer.writerows(self.rows) + except Exception as e: + logger.error(f"Error occurred while writing CSV: {e}") + raise + + def create_rows(self, inputs, delete_flag): + """Add new rows into the CSV, also replace missing values with empty strings and removes duplicate rows.""" + try: + new_rows = [self._normalize_row(row) for row in inputs] + + # Deduplicate: use tuple of values as unique key + if delete_flag: + existing = set({}) + else: + existing = { + tuple(row[col] for col in self.columns) for row in self.rows + } + for row in new_rows: + key = tuple(row[col] for col in self.columns) + if key not in existing: + self.rows.append(row) + existing.add(key) + + self._write_to_csv() + except Exception as e: + logger.error(f"Error occurred while adding new rows: {e}") + raise + + def delete_rows_by_key(self, key): + """Delete all rows where the 'key' column matches.""" + try: + self.rows = [ + row for row in self.rows if row["key"].strip() != str(key).strip() + ] + except Exception as e: + logger.error(f"Error occurred while deleting row by key: {e}") + raise diff --git a/splunk_connect_for_snmp/common/discovery_record.py b/splunk_connect_for_snmp/common/discovery_record.py new file mode 100644 index 000000000..75e2460bd --- /dev/null +++ b/splunk_connect_for_snmp/common/discovery_record.py @@ -0,0 +1,105 @@ +# Copyright 2021 Splunk Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from ipaddress import ip_network +from typing import List, Union + +from pydantic import BaseModel, validator + +from splunk_connect_for_snmp.common.hummanbool import human_bool + +DiscoveryStr = Union[None, str] +DiscoveryInt = Union[None, int] +DiscoveryBool = Union[None, bool] +DiscoveryList = Union[None, List[dict]] + + +class DiscoveryRecord(BaseModel): + discovery_name: DiscoveryStr + network_address: DiscoveryStr + address: DiscoveryStr + port: DiscoveryInt = 161 + version: DiscoveryStr + community: DiscoveryStr + secret: DiscoveryStr + security_engine: DiscoveryStr = "" + frequency: DiscoveryInt + delete_already_discovered: DiscoveryBool + device_rules: DiscoveryList + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + @validator("network_address", pre=True) + def network_address_validator(cls, value): + if value is None: + raise ValueError("field network address cannot be null") + else: + try: + ip_network(value, strict=False) + except ValueError: + raise ValueError(f"field network address must be an valid subnet") + + return value + + @validator("port", pre=True) + def port_validator(cls, value): + if value is None: + return 161 + if isinstance(value, int) and value >= 1 or value <= 65535: + return value + else: + raise ValueError("field port must be an integer between 1 and 65535") + + @validator("version", pre=True) + def version_validator(cls, value): + if value is None or value.strip() == "": + return "2c" + else: + if value not in ("1", "2c", "3"): + raise ValueError( + f"version out of range {value} accepted is 1 or 2c or 3" + ) + return value + + @validator("community", "secret", "security_engine", pre=True) + def community_secret_security_engine_validator(cls, value): + if value is None or (isinstance(value, str) and value.strip() == ""): + return None + return value + + @validator("frequency", pre=True) + def frequency_validator(cls, value): + if value is None: + return 86400 + elif value < 21600: + return 21600 + + return value + + @validator("device_rules", pre=True) + def device_rules_validator(cls, value): + if value is None or (isinstance(value, list) and value == []): + return None + return value + + @validator("delete_already_discovered", pre=True) + def delete_already_discovered_validator(cls, value): + if value is None: + return False + return human_bool(value) + + def asdict(self) -> dict: + return self.dict() diff --git a/splunk_connect_for_snmp/common/task_generator.py b/splunk_connect_for_snmp/common/task_generator.py index bb2dc59ca..0b7481011 100644 --- a/splunk_connect_for_snmp/common/task_generator.py +++ b/splunk_connect_for_snmp/common/task_generator.py @@ -132,3 +132,28 @@ def run_immediately(self): if self.schedule_period > 300: return True return False + + +class DiscoveryTaskGenerator(TaskGenerator): + def __init__(self, discovery_record, app): + super().__init__( + target=discovery_record.network_address, + schedule_period=discovery_record.frequency, + app=app, + ) + self.discovery_record = discovery_record + self.discovery_name = discovery_record.discovery_name + self.DISCOVERY_CHAIN_OF_TASK = { + "queue": "discovery", + "expires": CHAIN_OF_TASKS_EXPIRY_TIME, + } + + def generate_task_definition(self): + task_data = super().generate_task_definition() + name = f"sc4snmp;{self.discovery_name};discovery" + task_data["name"] = name + task_data["task"] = "splunk_connect_for_snmp.discovery.tasks.discovery" + task_data["run_immediately"] = True + task_data["options"] = self.DISCOVERY_CHAIN_OF_TASK + task_data["kwargs"] = self.discovery_record.dict() + return task_data diff --git a/splunk_connect_for_snmp/customtaskmanager.py b/splunk_connect_for_snmp/customtaskmanager.py index f785db0dd..8df3c74d1 100644 --- a/splunk_connect_for_snmp/customtaskmanager.py +++ b/splunk_connect_for_snmp/customtaskmanager.py @@ -57,6 +57,7 @@ def did_expiry_time_change(self, new_expiry_time): if previous_expiry_time is not None and previous_expiry_time != new_expiry_time: self.delete_all_walk_tasks() self.delete_all_poll_tasks() + self.delete_all_discovery_tasks() expiry_time_changed = True return expiry_time_changed @@ -70,6 +71,12 @@ def delete_all_walk_tasks(self): "splunk_connect_for_snmp.snmp.tasks.walk", "delete_all_walk_tasks" ) + def delete_all_discovery_tasks(self): + self.__delete_all_tasks_of_type( + "splunk_connect_for_snmp.discovery.tasks.discovery", + "delete_all_discovery_tasks", + ) + def rerun_all_walks(self): periodic_tasks = RedBeatSchedulerEntry.get_schedules() for periodic_document in periodic_tasks: diff --git a/splunk_connect_for_snmp/discovery/discovery_manager.py b/splunk_connect_for_snmp/discovery/discovery_manager.py new file mode 100644 index 000000000..b5a34c773 --- /dev/null +++ b/splunk_connect_for_snmp/discovery/discovery_manager.py @@ -0,0 +1,136 @@ +import copy +import fnmatch +import ipaddress +import os +import re +from concurrent.futures import ThreadPoolExecutor, as_completed + +from celery import Task +from celery.utils.log import get_task_logger +from filelock import FileLock +from pysnmp.hlapi import ContextData, ObjectIdentity, ObjectType, SnmpEngine, getCmd + +from splunk_connect_for_snmp.common.csv_record_manager import CSVRecordManager +from splunk_connect_for_snmp.common.discovery_record import DiscoveryRecord +from splunk_connect_for_snmp.snmp.auth import get_auth, setup_transport_target + +logger = get_task_logger(__name__) + +DISCOVERY_FOLDER_PATH = os.getenv("DISCOVERY_FOLDER_PATH", "/app/discovery") +DISCOVERY_CSV_PATH = os.path.join(DISCOVERY_FOLDER_PATH, "discovery_devices.csv") +DISCOVERY_LOCK_PATH = os.path.join(DISCOVERY_FOLDER_PATH, "discovery_devices.lock") + + +class Discovery(Task): + def __init__(self): + self.snmp_engine = SnmpEngine() + + def get_host_list(self, subnet): + """Get host list""" + try: + network = ipaddress.ip_network(subnet, strict=False) + return list(str(ip) for ip in network.hosts()) + except Exception as e: + logger.error(f"Error occured while finding active hosts: {e}") + raise + + def check_snmp_device(self, ip, discovery_record: DiscoveryRecord): + """Check if an SNMP device responds at the given IP.""" + discovery_record.address = ip + auth_data = get_auth(logger, discovery_record, SnmpEngine()) + transport_target = setup_transport_target(discovery_record) + iterator = getCmd( + SnmpEngine(), + auth_data, + transport_target, + ContextData(), + ObjectType(ObjectIdentity("SNMPv2-MIB", "sysDescr", 0)), + ) + + error_indication, error_status, error_index, var_binds = next(iterator) + + if not error_indication and error_status == 0: + group_name = "default_group" + _, value = var_binds[0] + if isinstance(discovery_record.device_rules, list): + for device_rule in discovery_record.device_rules: + regex_pattern = fnmatch.translate(device_rule["patterns"]) + if re.search(regex_pattern, value.prettyPrint(), re.IGNORECASE): + group_name = device_rule["group"] + break + return { + "key": discovery_record.discovery_name, + "ip": ip, + "subnet": discovery_record.network_address, + "group": group_name, + "version": discovery_record.version, + "port": discovery_record.port, + "secret": discovery_record.secret, + "community": discovery_record.community, + } + return None + + def discover_snmp_devices_details( + self, ip_list: list, discovery_record: DiscoveryRecord, max_threads=5 + ): + """Scan subnet for SNMP-enabled devices using multithreading.""" + devices_detail = [] + + with ThreadPoolExecutor(max_workers=max_threads) as executor: + future_to_ip = { + executor.submit( + self.check_snmp_device, + ip=ip, + discovery_record=copy.deepcopy(discovery_record), + ): ip + for ip in ip_list + } + + for _, future in enumerate(as_completed(future_to_ip), start=1): + ip = future_to_ip[future] + try: + result = future.result() + if result: + devices_detail.append(result) + logger.info( + f"SNMP device found: {result}. Device is from discovery: {discovery_record.discovery_name}" + ) + else: + logger.info(f"SNMP not enabled on the device: {ip}") + except Exception as e: + logger.error( + f"Snmp check for device {ip} generated an exception : {e}" + ) + + return devices_detail + + def add_devices_detail_to_csv( + self, snmp_devices_detail, delete_flag, dicovery_name + ): + """Add snmp devices detail to CSV""" + lock = FileLock(DISCOVERY_LOCK_PATH) + with lock: + csv_service = CSVRecordManager(DISCOVERY_CSV_PATH) + if delete_flag is True: + csv_service.delete_rows_by_key(dicovery_name) + csv_service.create_rows(snmp_devices_detail, delete_flag) + + def do_work(self, discovery_record: DiscoveryRecord) -> list: + try: + host_list = self.get_host_list( + discovery_record.network_address, + ) + logger.info(f"Number of Active hosts: {len(host_list)}") + snmp_devices_detail = self.discover_snmp_devices_details( + host_list, discovery_record, max_threads=10 + ) + self.add_devices_detail_to_csv( + snmp_devices_detail, + discovery_record.delete_already_discovered, + discovery_record.discovery_name, + ) + + return snmp_devices_detail + except Exception as e: + logger.error(f"Error occurred while finding SMNP enabled device: {e}") + raise diff --git a/splunk_connect_for_snmp/discovery/loader.py b/splunk_connect_for_snmp/discovery/loader.py new file mode 100644 index 000000000..98c859f73 --- /dev/null +++ b/splunk_connect_for_snmp/discovery/loader.py @@ -0,0 +1,91 @@ +import ipaddress +import logging +import os +import sys +from contextlib import suppress + +import yaml + +from splunk_connect_for_snmp import customtaskmanager +from splunk_connect_for_snmp.common.customised_json_formatter import ( + CustomisedJSONFormatter, +) +from splunk_connect_for_snmp.common.discovery_record import DiscoveryRecord +from splunk_connect_for_snmp.common.task_generator import DiscoveryTaskGenerator +from splunk_connect_for_snmp.poller import app + +with suppress(ImportError, OSError): + from dotenv import load_dotenv + + load_dotenv() + +DISCOVERY_CONFIG_PATH = os.getenv( + "DISCOVERY_CONFIG_PATH", "/app/discovery/discovery-config.yaml" +) +CHAIN_OF_TASKS_EXPIRY_TIME = os.getenv("CHAIN_OF_TASKS_EXPIRY_TIME", "60") +LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO") + +formatter = CustomisedJSONFormatter() + +logger = logging.getLogger(__name__) +logger.setLevel(LOG_LEVEL) + +# writing to stdout +handler = logging.StreamHandler(sys.stdout) +handler.setLevel(LOG_LEVEL) +handler.setFormatter(formatter) +logger.addHandler(handler) + + +def autodiscovery_task_definition(discovery_record, app): + discovery_definition = DiscoveryTaskGenerator( + discovery_record=discovery_record, app=app + ) + task_config = discovery_definition.generate_task_definition() + return task_config + + +def check_ipv6(subnet): + network = ipaddress.ip_network(subnet, strict=False) + return isinstance(network, ipaddress.IPv6Network) + + +def load(): + try: + with open(DISCOVERY_CONFIG_PATH, encoding="utf-8") as file: + config_runtime = yaml.safe_load(file) + ipv6_enabled = config_runtime.get("ipv6Enabled", False) + autodiscovery = config_runtime.get("autodiscovery", {}) + periodic_obj = customtaskmanager.CustomPeriodicTaskManager() + expiry_time_changed = periodic_obj.did_expiry_time_change( + CHAIN_OF_TASKS_EXPIRY_TIME + ) + if expiry_time_changed: + logger.info( + f"Task expiry time was modified, generating new tasks for discovery" + ) + + for key, value in autodiscovery.items(): + value["discovery_name"] = key + discovery_record = DiscoveryRecord(**value) + is_ipv6 = check_ipv6(value["network_address"]) + if not is_ipv6 or (is_ipv6 and ipv6_enabled): + logger.info(f"Adding the task for {key}") + task_config = autodiscovery_task_definition( + discovery_record=discovery_record, app=app + ) + periodic_obj.manage_task(**task_config) + else: + logger.info( + f"Skipping task for the discovery: {key} because IPv6 is disabled." + ) + return 0 + except Exception as e: + logger.error("Error occured while creating the task : {e}") + raise + + +if __name__ == "__main__": + r = load() + if r: + sys.exit(0) diff --git a/splunk_connect_for_snmp/discovery/tasks.py b/splunk_connect_for_snmp/discovery/tasks.py new file mode 100644 index 000000000..a6eeed88c --- /dev/null +++ b/splunk_connect_for_snmp/discovery/tasks.py @@ -0,0 +1,14 @@ +from celery import shared_task +from celery.utils.log import get_task_logger + +from splunk_connect_for_snmp.common.discovery_record import DiscoveryRecord +from splunk_connect_for_snmp.discovery.discovery_manager import Discovery + +logger = get_task_logger(__name__) + + +@shared_task(bind=True, base=Discovery) +def discovery(self, **kwargs) -> dict: + discovery_record = DiscoveryRecord(**kwargs) + result = self.do_work(discovery_record) + return {"snmp_device_details": result} diff --git a/splunk_connect_for_snmp/poller.py b/splunk_connect_for_snmp/poller.py index 099cc3c0b..f9488269f 100644 --- a/splunk_connect_for_snmp/poller.py +++ b/splunk_connect_for_snmp/poller.py @@ -50,5 +50,6 @@ "splunk_connect_for_snmp.inventory", "splunk_connect_for_snmp.snmp", "splunk_connect_for_snmp.splunk", + "splunk_connect_for_snmp.discovery", ] ) diff --git a/splunk_connect_for_snmp/snmp/auth.py b/splunk_connect_for_snmp/snmp/auth.py index cf89bc297..9d3ffe0ac 100644 --- a/splunk_connect_for_snmp/snmp/auth.py +++ b/splunk_connect_for_snmp/snmp/auth.py @@ -30,15 +30,20 @@ from pysnmp.proto.api.v2c import OctetString from pysnmp.smi.rfc1902 import ObjectIdentity, ObjectType +from splunk_connect_for_snmp.common.discovery_record import DiscoveryRecord from splunk_connect_for_snmp.common.hummanbool import human_bool from splunk_connect_for_snmp.common.inventory_record import InventoryRecord from splunk_connect_for_snmp.snmp.const import AuthProtocolMap, PrivProtocolMap from splunk_connect_for_snmp.snmp.exceptions import SnmpActionError UDP_CONNECTION_TIMEOUT = int(os.getenv("UDP_CONNECTION_TIMEOUT", 1)) +UDP_CONNECTION_RETRIES = int(os.getenv("UDP_CONNECTION_RETRIES", 5)) IPv6_ENABLED = human_bool(os.getenv("IPv6_ENABLED", "false").lower()) +RecordType = Union[DiscoveryRecord, InventoryRecord] + + def get_secret_value( location: str, key: str, default: str = "", required: bool = False ) -> str: @@ -56,10 +61,10 @@ def get_secret_value( # To discover remote SNMP EngineID we will tap on SNMP engine inner workings # by setting up execution point observer setup on INTERNAL class PDU processing # -def get_security_engine_id(logger, ir: InventoryRecord, snmp_engine: SnmpEngine): +def get_security_engine_id(logger, rt: RecordType, snmp_engine: SnmpEngine): observer_context: Dict[Any, Any] = {} - transport_target = setup_transport_target(ir) + transport_target = setup_transport_target(rt) # Register a callback to be invoked at specified execution point of # SNMP Engine and passed local variables at execution point's local scope @@ -84,27 +89,33 @@ def get_security_engine_id(logger, ir: InventoryRecord, snmp_engine: SnmpEngine) # See if our SNMP engine received REPORT PDU containing securityEngineId security_engine_id = fetch_security_engine_id( - observer_context, error_indication, ir.address + observer_context, error_indication, rt.address ) - logger.debug(f"securityEngineId={security_engine_id} for device {ir.address}") + logger.debug(f"securityEngineId={security_engine_id} for device {rt.address}") return security_engine_id -def setup_transport_target(ir): - ip = get_ip_from_socket(ir) if IPv6_ENABLED else ir.address +def setup_transport_target(rt): + ip = get_ip_from_socket(rt) if IPv6_ENABLED else rt.address if IPv6_ENABLED and ip_address(ip).version == 6: return Udp6TransportTarget( - (ir.address, ir.port), timeout=UDP_CONNECTION_TIMEOUT + (rt.address, rt.port), + timeout=UDP_CONNECTION_TIMEOUT, + retries=UDP_CONNECTION_RETRIES, ) - return UdpTransportTarget((ir.address, ir.port), timeout=UDP_CONNECTION_TIMEOUT) + return UdpTransportTarget( + (rt.address, rt.port), + timeout=UDP_CONNECTION_TIMEOUT, + retries=UDP_CONNECTION_RETRIES, + ) -def get_ip_from_socket(ir): +def get_ip_from_socket(rt): # Example of response from getaddrinfo # [(< AddressFamily.AF_INET6: 10 >, < SocketKind.SOCK_STREAM: 1 >, 6, '', ('2607:f8b0:4004:c09::64', 161, 0, 0)), # (< AddressFamily.AF_INET: 2 >, < SocketKind.SOCK_STREAM: 1 >, 6, '', ('142.251.16.139', 161))] - return socket.getaddrinfo(ir.address, ir.port)[0][4][0] + return socket.getaddrinfo(rt.address, rt.port)[0][4][0] def fetch_security_engine_id(observer_context, error_indication, ipaddress): @@ -116,8 +127,8 @@ def fetch_security_engine_id(observer_context, error_indication, ipaddress): ) -def get_auth_v3(logger, ir: InventoryRecord, snmp_engine: SnmpEngine) -> UsmUserData: - location = os.path.join("secrets/snmpv3", ir.secret) # type: ignore +def get_auth_v3(logger, rt: RecordType, snmp_engine: SnmpEngine) -> UsmUserData: + location = os.path.join("secrets/snmpv3", rt.secret) # type: ignore if os.path.exists(location): username = get_secret_value(location, "userName", required=True) @@ -140,14 +151,14 @@ def get_auth_v3(logger, ir: InventoryRecord, snmp_engine: SnmpEngine) -> UsmUser get_secret_value(location, "privKeyType", required=False, default="0") ) if ( - isinstance(ir.security_engine, str) - and ir.security_engine != "" - and not ir.security_engine.isdigit() + isinstance(rt.security_engine, str) + and rt.security_engine != "" + and not rt.security_engine.isdigit() ): - security_engine_id = OctetString(hexValue=ir.security_engine) + security_engine_id = OctetString(hexValue=rt.security_engine) logger.debug(f"Security eng from profile {security_engine_id}") else: - security_engine_id = get_security_engine_id(logger, ir, snmp_engine) + security_engine_id = get_security_engine_id(logger, rt, snmp_engine) logger.debug(f"Security eng dynamic {security_engine_id}") security_name = None @@ -167,23 +178,23 @@ def get_auth_v3(logger, ir: InventoryRecord, snmp_engine: SnmpEngine) -> UsmUser ) else: - raise FileNotFoundError(f"invalid username from secret {ir.secret}") + raise FileNotFoundError(f"invalid username from secret {rt.secret}") -def get_auth_v2c(ir: InventoryRecord) -> CommunityData: - return CommunityData(ir.community, mpModel=1) +def get_auth_v2c(rt: RecordType) -> CommunityData: + return CommunityData(rt.community, mpModel=1) -def get_auth_v1(ir: InventoryRecord) -> CommunityData: - return CommunityData(ir.community, mpModel=0) +def get_auth_v1(rt: RecordType) -> CommunityData: + return CommunityData(rt.community, mpModel=0) def get_auth( - logger, ir: InventoryRecord, snmp_engine: SnmpEngine + logger, rt: RecordType, snmp_engine: SnmpEngine ) -> Union[UsmUserData, CommunityData]: - if ir.version == "1": - return get_auth_v1(ir) - elif ir.version == "2c": - return get_auth_v2c(ir) + if rt.version == "1": + return get_auth_v1(rt) + elif rt.version == "2c": + return get_auth_v2c(rt) else: - return get_auth_v3(logger, ir, snmp_engine) + return get_auth_v3(logger, rt, snmp_engine)