chore: updating integeration tests docker env[run-int-tests]

Author: cwadhwani-splunk

integration_tests/.env

@@ -5,6 +5,7 @@ SCHEDULER_CONFIG_FILE_ABSOLUTE_PATH=
 TRAPS_CONFIG_FILE_ABSOLUTE_PATH=
 INVENTORY_FILE_ABSOLUTE_PATH=
 COREFILE_ABS_PATH=
+DISCOVERY_CONFIG_FILE_ABSOLUTE_PATH=
 COREDNS_ADDRESS=172.28.0.255
 COREDNS_ADDRESS_IPv6=fd02:0:0:0:7fff:ffff:ffff:ffff
 SC4SNMP_VERSION=latest
@@ -35,6 +36,11 @@ SPLUNK_HEC_PATH=/services/collector
 SPLUNK_AGGREGATE_TRAPS_EVENTS=false
 IGNORE_EMPTY_VARBINDS=false
 
+# Discovery
+DISCOVERY_ENABLE=true
+DISCOVERY_LOG_LEVEL=INFO
+DISCOVERY_PATH=
+
 # Workers configration
 WALK_RETRY_MAX_INTERVAL=180
 WALK_MAX_RETRIES=5
@@ -75,6 +81,15 @@ WORKER_TRAP_MEMORY_LIMIT=500M
 WORKER_TRAP_CPU_RESERVATIONS=0.5
 WORKER_TRAP_MEMORY_RESERVATIONS=250M
 
+# Worker Discovery
+WORKER_DISCOVERY_CONCURRENCY=4
+PREFETCH_DISCOVERY_COUNT=30
+WORKER_DISCOVERY_REPLICAS=1
+WORKER_DISCOVERY_CPU_LIMIT=1
+WORKER_DISCOVERY_MEMORY_LIMIT=500M
+WORKER_DISCOVERY_CPU_RESERVATIONS=0.5
+WORKER_DISCOVERY_MEMORY_RESERVATIONS=250M
+
 # Inventory configuration
 INVENTORY_LOG_LEVEL=INFO
 CHAIN_OF_TASKS_EXPIRY_TIME=500

integration_tests/automatic_setup_compose.sh

@@ -104,32 +104,40 @@ cp ../docker_compose/* .
 SCHEDULER_CONFIG_FILE="scheduler-config.yaml"
 TRAPS_CONFIG_FILE="traps-config.yaml"
 INVENTORY_FILE="inventory-tests.csv"
+DISCOVERY_CONFIG_FILE="discovery-config.yaml"
 COREFILE="Corefile"
+DISCOVERY_FOLDER="discovery"
 
 # Get the absolute paths of the files
 SCHEDULER_CONFIG_FILE_ABSOLUTE_PATH=$(realpath "$SCHEDULER_CONFIG_FILE")
 TRAPS_CONFIG_FILE_ABSOLUTE_PATH=$(realpath "$TRAPS_CONFIG_FILE")
+DISCOVERY_CONFIG_FILE_ABSOLUTE_PATH=$(realpath "$DISCOVERY_CONFIG_FILE")
 INVENTORY_FILE_ABSOLUTE_PATH=$(realpath "$INVENTORY_FILE")
 COREFILE_ABS_PATH=$(realpath "$COREFILE")
 SPLUNK_HEC_HOST=$(hostname -I | cut -d " " -f1)
 SPLUNK_HEC_TOKEN=$(cat hec_token)
+DISCOVERY_PATH=$(realpath "$DISCOVERY_FOLDER")
 
 # Temporary file to store the updated .env content
 TEMP_ENV_FILE=".env.tmp"
 
 # Update or add the variables in the .env file
 awk -v scheduler_path="$SCHEDULER_CONFIG_FILE_ABSOLUTE_PATH" \
     -v traps_path="$TRAPS_CONFIG_FILE_ABSOLUTE_PATH" \
+    -v discovery_config_path="$DISCOVERY_CONFIG_FILE_ABSOLUTE_PATH" \
     -v inventory_path="$INVENTORY_FILE_ABSOLUTE_PATH" \
     -v corefile_path="$COREFILE_ABS_PATH" \
+    -v discovery_path="$DISCOVERY_PATH" \
     -v splunk_hec_host="$SPLUNK_HEC_HOST" \
     -v splunk_hec_token="$SPLUNK_HEC_TOKEN" \
     '
     BEGIN {
         updated["SCHEDULER_CONFIG_FILE_ABSOLUTE_PATH"] = 0;
-        updated["TRAPS_CONFIG_FILE_ABSOLUTE_PATH"] = 0;
+        updated["TRAPS_CONFIG_FILE_ABSOLUTE_PATH"] = 0
+        updated["DICOVERY_CONFIG_FILE_ABSOLUTE_PATH"] = 0;
         updated["INVENTORY_FILE_ABSOLUTE_PATH"] = 0;
         updated["COREFILE_ABS_PATH"] = 0;
+        updated["DISCOVERY_PATH"] = 0;
         updated["SPLUNK_HEC_HOST"] = 0;
         updated["SPLUNK_HEC_TOKEN"] = 0;
     }
@@ -140,12 +148,18 @@ awk -v scheduler_path="$SCHEDULER_CONFIG_FILE_ABSOLUTE_PATH" \
         } else if ($1 == "TRAPS_CONFIG_FILE_ABSOLUTE_PATH=") {
             print "TRAPS_CONFIG_FILE_ABSOLUTE_PATH=" traps_path;
             updated["TRAPS_CONFIG_FILE_ABSOLUTE_PATH"] = 1;
+        } else if ($1 == "DISCOVERY_CONFIG_FILE_ABSOLUTE_PATH=") {
+            print "DISCOVERY_CONFIG_FILE_ABSOLUTE_PATH=" discovery_config_path;
+            updated["DISCOVERY_CONFIG_FILE_ABSOLUTE_PATH"] = 1;
         } else if ($1 == "INVENTORY_FILE_ABSOLUTE_PATH=") {
             print "INVENTORY_FILE_ABSOLUTE_PATH=" inventory_path;
             updated["INVENTORY_FILE_ABSOLUTE_PATH"] = 1;
         } else if ($1 == "COREFILE_ABS_PATH=") {
             print "COREFILE_ABS_PATH=" corefile_path;
             updated["COREFILE_ABS_PATH"] = 1;
+        } else if ($1 == "DISCOVERY_PATH=") {
+            print "DISCOVERY_PATH=" discovery_path;
+            updated["DISCOVERY_PATH"] = 1;
         } else if ($1 == "SPLUNK_HEC_HOST=") {
             print "SPLUNK_HEC_HOST=" splunk_hec_host;
             updated["SPLUNK_HEC_HOST"] = 1;
@@ -163,12 +177,18 @@ awk -v scheduler_path="$SCHEDULER_CONFIG_FILE_ABSOLUTE_PATH" \
         if (updated["TRAPS_CONFIG_FILE_ABSOLUTE_PATH"] == 0) {
             print "TRAPS_CONFIG_FILE_ABSOLUTE_PATH=" traps_path;
         }
+        if (updated["DISCOVERY_CONFIG_FILE_ABSOLUTE_PATH"] == 0) {
+            print "DISCOVERY_CONFIG_FILE_ABSOLUTE_PATH=" discovery_config_path;
+        }
         if (updated["INVENTORY_FILE_ABSOLUTE_PATH"] == 0) {
             print "INVENTORY_FILE_ABSOLUTE_PATH=" inventory_path;
         }
         if (updated["COREFILE_ABS_PATH"] == 0) {
             print "COREFILE_ABS_PATH=" corefile_path;
         }
+        if (updated["DISCOVERY_PATH"] == 0) {
+            print "DISCOVERY_PATH=" discovery_path;
+        }
         if (updated["SPLUNK_HEC_HOST"] == 0) {
             print "SPLUNK_HEC_HOST=" splunk_hec_host;
         }

integration_tests/discovery-config.yaml

@@ -95,10 +95,8 @@ def discover_snmp_devices_details(
                         logger.info(
                             f"SNMP device found: {result}. Device is from discovery: {discovery_record.discovery_name}"
                         )
-                    else: 
-                        logger.info(
-                            f"SNMP not enabled on the device: {ip}"
-                        )
+                    else:
+                        logger.info(f"SNMP not enabled on the device: {ip}")
                 except Exception as e:
                     logger.error(
                         f"Snmp check for device {ip} generated an exception : {e}"