DPDV-4287: Pass the proxy settings (#92)

* DPDV-4287: Pass the proxy settings

* Include proxies also for get and delete queries

* Fix the synchronization script and update the client as well

* Fix the conversion

* Fix the proxy configuration

* Store the release and output folder

* Make HTTP protocol explicit in the UI

Author: martin-majlis-s1

.github/workflows/validate.yaml

@@ -40,3 +40,19 @@ jobs:
             --input TA_dataset \
             --output output \
             --release release
+
+      - name: Store Release Folder
+        uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: release
+          path: release
+          retention-days: 30
+
+      - name: Store Output Folder
+        uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: output
+          path: output
+          retention-days: 30

CONTRIBUTING.md

@@ -125,10 +125,14 @@ Note that build cleans previously created configuration. To prevent removal of c
 * Remove Splunk container - `make docker-splunk-remove`
 * Restore configuration - `make dev-config-backup`
 * Backup configuration - `make dev-config-restore` - it's not clear whether it really works
-* Tail Splunk logs - Splunkd - `make docker-tail-logs-splunk`
-* Tail Splunk logs - Python - `make docker-tail-logs-python`
-* Tail Splunk logs - Inputs - `make docker-tail-logs-inputs`
+* To see all available logs - `make docker-splunk-list-logs`
+* To see particular log, you may use - `make docker-splunk-tail-log LOG_NAME=log-file`
+  * Logs related to Splunk Python - `make docker-splunk-tail-logs-python` calls `make docker-splunk-tail-log LOG_NAME=python.log`
+  * Logs related to Search command - `make docker-splunk-tail-logs-app-search-command` calls `make docker-splunk-tail-log LOG_NAME="TA_dataset_search_command.log"`
 
+#### Where are errors:
+
+* `search_messages.log` - error message that is shown in the UI, no stack trace :/
 # E2E Testing
 
 We are using Playwright - https://playwright.dev/

Makefile

@@ -51,26 +51,34 @@ docker-splunk-show-app:
 		sudo -u splunk \
 		ls -l /opt/splunk/etc/apps/TA_dataset/
 
-.PHONY: docker-splunk-tail-logs-splunkd
-docker-splunk-tail-logs-splunkd:
+.PHONY: docker-splunk-list-logs
+docker-splunk-list-logs:
 	docker exec $(CONTAINER_NAME) \
 		sudo -u splunk \
-		tail -f \
-			/opt/splunk/var/log/splunk/splunkd.log
+		ls -lrt \
+			/opt/splunk/var/log/splunk/
+
+.PHONY: docker-splunk-tail-log
+docker-splunk-tail-log:
+	docker exec $(CONTAINER_NAME) \
+		bash -l -c "sudo -u splunk tail -f /opt/splunk/var/log/splunk/${LOG_NAME}"
+
+.PHONY: docker-splunk-tail-logs-splunkd
+docker-splunk-tail-logs-splunkd:
+	make docker-splunk-tail-log LOG_NAME=splunkd.log
 
 .PHONY: docker-splunk-tail-logs-python
 docker-splunk-tail-logs-python:
-	docker exec $(CONTAINER_NAME) \
-		sudo -u splunk \
-		tail -f \
-			/opt/splunk/var/log/splunk/python.log
+	make docker-splunk-tail-log LOG_NAME=python.log
 
-.PHONY: docker-splunk-tail-logs-input
-docker-splunk-tail-logs-input:
-	docker exec $(CONTAINER_NAME) \
-		sudo -u splunk \
-		tail -f \
-			/opt/splunk/var/log/splunk/TA_dataset_input.log
+# TODO: Figure out, how to make this work!
+.PHONY: docker-splunk-tail-logs-app-all
+docker-splunk-tail-logs-app-all:
+	make docker-splunk-tail-log LOG_NAME="TA_dataset*"
+
+.PHONY: docker-splunk-tail-logs-app-search-command
+docker-splunk-tail-logs-app-search-command:
+	make docker-splunk-tail-log LOG_NAME="TA_dataset_search_command.log"
 
 .PHONY: docker-splunk-tail-logs
 docker-splunk-tail-logs-count:
@@ -101,7 +109,7 @@ inspect:
 	splunk-appinspect inspect TA_dataset --included-tags splunk_appinspect
 
 .PHONY: pack
-pack:
+pack: clean
 	find $(SOURCE_PACKAGE) -name __pycache__ -exec rm -rfv {} \;
 	version=$$(jq -r '.meta.version' globalConfig.json) && \
 	scripts/pack.sh \
@@ -137,6 +145,7 @@ dev-config-restore:
 dev-update-source:
 	rsync -av $(SOURCE_PACKAGE)/bin/ $(OUTPUT_PACKAGE)/bin/
 	rsync -av $(SOURCE_PACKAGE)/default/ $(OUTPUT_PACKAGE)/default/
+	rsync -av $(SOURCE_PACKAGE)/lib/dataset_query_api_client/ $(OUTPUT_PACKAGE)/lib/dataset_query_api_client/
 
 dev-install-dependencies-pack:
 	pip install --upgrade-strategy only-if-needed -r requirements-pack.txt
@@ -162,3 +171,7 @@ e2e-test-headed:
 	npm run playwright:headed
 e2e-test-ui:
 	npm run playwright:ui
+
+.PHONY: clean
+clean:
+	find $(SOURCE_PACKAGE) -name '.DS_Store' -exec rm -rfv {} \;

TA_dataset/bin/dataset_api.py

@@ -24,9 +24,26 @@
 )
 
 
+# TODO: Convert to the expected format
+# https://www.python-httpx.org/advanced/#http-proxying
+def convert_proxy(proxy):
+    if not proxy:
+        return {}
+    new_proxy = {}
+    if "http" in proxy:
+        new_proxy["http://"] = proxy["http"]
+    if "https" in proxy:
+        new_proxy["https://"] = proxy["https"]
+    return new_proxy
+
+
 # Executes Dataset LongRunningQuery for log events
-def ds_lrq_log_query(base_url, api_key, start_time, end_time, filter_expr, limit):
-    client = AuthenticatedClient(base_url=base_url, token=api_key)
+def ds_lrq_log_query(
+    base_url, api_key, start_time, end_time, filter_expr, limit, proxy
+):
+    client = AuthenticatedClient(
+        base_url=base_url, token=api_key, proxy=convert_proxy(proxy)
+    )
     body = PostQueriesLaunchQueryRequestBody(
         query_type=PostQueriesLaunchQueryRequestBodyQueryType.LOG,
         start_time=start_time,
@@ -37,8 +54,10 @@ def ds_lrq_log_query(base_url, api_key, start_time, end_time, filter_expr, limit
 
 
 # Executes Dataset LongRunningQuery using PowerQuery language
-def ds_lrq_power_query(base_url, api_key, start_time, end_time, query):
-    client = AuthenticatedClient(base_url=base_url, token=api_key)
+def ds_lrq_power_query(base_url, api_key, start_time, end_time, query, proxy):
+    client = AuthenticatedClient(
+        base_url=base_url, token=api_key, proxy=convert_proxy(proxy)
+    )
     body = PostQueriesLaunchQueryRequestBody(
         query_type=PostQueriesLaunchQueryRequestBodyQueryType.PQ,
         start_time=start_time,
@@ -50,9 +69,11 @@ def ds_lrq_power_query(base_url, api_key, start_time, end_time, query):
 
 # Executes Dataset LongRunningQuery to fetch facet values
 def ds_lrq_facet_values(
-    base_url, api_key, start_time, end_time, filter, name, max_values
+    base_url, api_key, start_time, end_time, filter, name, max_values, proxy
 ):
-    client = AuthenticatedClient(base_url=base_url, token=api_key)
+    client = AuthenticatedClient(
+        base_url=base_url, token=api_key, proxy=convert_proxy(proxy)
+    )
     body = PostQueriesLaunchQueryRequestBody(
         query_type=PostQueriesLaunchQueryRequestBodyQueryType.FACET_VALUES,
         start_time=start_time,
@@ -66,7 +87,9 @@ def ds_lrq_facet_values(
 
 # Executes LRQ run loop of launch-ping-remove API requests until the query completes
 # with a result
-def ds_lrq_run_loop(client, body: PostQueriesLaunchQueryRequestBody):
+def ds_lrq_run_loop(
+    client: AuthenticatedClient, body: PostQueriesLaunchQueryRequestBody
+):
     body.query_priority = PostQueriesLaunchQueryRequestBodyQueryPriority.HIGH
     response = post_queries.sync_detailed(client=client, json_body=body)
     result = response.parsed

TA_dataset/bin/dataset_common.py

@@ -1,12 +1,13 @@
 # -*- coding: utf-8 -*-
 import json
+import logging
 import os.path as op
 import sys
 import time
 
 # adjust paths to make the Splunk app working
 import import_declare_test  # noqa: F401
-from solnlib import conf_manager
+from solnlib import conf_manager, log
 
 APP_NAME = __file__.split(op.sep)[-3]
 CONF_NAME = "ta_dataset"
@@ -28,6 +29,15 @@ def get_url(base_url, ds_method):
     return base_url + "/api/" + ds_api_endpoint
 
 
+# returns logger that logs data into file
+# /opt/splunk/var/log/splunk/${APP_NAME}/${suffix}
+def get_logger(session_key, suffix: str):
+    logger = log.Logs().get_logger("{}_{}".format(APP_NAME, suffix))
+    log_level = get_log_level(session_key, logging)
+    logger.setLevel(log_level)
+    return logger
+
+
 # one conf manager to rule them all
 def get_conf_manager(session_key, logger):
     try:
@@ -84,7 +94,6 @@ def get_proxy(session_key, logger):
             # MM: it does not have key `proxy_enabled, it has key - disabled
             #  {'disabled': '0', 'eai:appName': 'TA_dataset' ...
             proxy_details = cfm.get_conf(CONF_NAME + "_settings").get("proxy")
-            logger.info("MM PROXY: " + repr(proxy_details))
             proxy_enabled = proxy_details.get("proxy_enabled", 0)
         except Exception as e:
             logger.debug("No proxy information defined: {}".format(e))
@@ -93,30 +102,30 @@ def get_proxy(session_key, logger):
         if int(proxy_enabled) == 0:
             return None
         else:
-            proxy_url = proxy_details.get("proxy_url")
+            proxy_type = proxy_details.get("proxy_type")
+            proxy_host = proxy_details.get("proxy_url")
             proxy_port = proxy_details.get("proxy_port")
             proxy_username = proxy_details.get("proxy_username")
             proxy_password = proxy_details.get("proxy_password")
-            proxy_type = proxy_details.get("proxy_type")
             proxies = {}
-            if proxy_username and proxy_password:
-                proxies["http"] = (
-                    proxy_username
-                    + ":"
-                    + proxy_password
-                    + "@"
-                    + proxy_url
-                    + ":"
-                    + proxy_port
-                )
-            elif proxy_username:
-                proxies["http"] = proxy_username + "@" + proxy_url + ":" + proxy_port
+            proxy_url = ""
+            if proxy_type:
+                proxy_url += proxy_type
             else:
-                proxies["http"] = proxy_url + ":" + proxy_port
-            if proxy_type and proxy_type != "http":
-                proxies["http"] = proxy_type + "://" + proxies["http"]
-
+                proxy_url += "http"
+            proxy_url += "://"
+            if proxy_username:
+                proxy_url += proxy_username
+            if proxy_password:
+                proxy_url += ":" + proxy_password
+            if proxy_username:
+                proxy_url += "@"
+            proxy_url += proxy_host
+            proxy_url += ":" + proxy_port
+
+            proxies["http"] = proxy_url
             proxies["https"] = proxies["http"]
+
             return proxies
 
     except Exception as e:

TA_dataset/bin/dataset_query.py

@@ -14,13 +14,12 @@
 from dataset_api import build_payload, get_maxcount, parse_query, query_api_max
 from dataset_common import (
     get_acct_info,
-    get_log_level,
+    get_logger,
     get_proxy,
     get_url,
     relative_to_epoch,
 )
 from dataset_query_api_client.client import get_user_agent
-from solnlib import log
 from solnlib.modular_input import checkpointer
 from splunklib import modularinput as smi
 
@@ -115,11 +114,7 @@ def stream_events(self, inputs, ew):
 
         # Generate logger with input name
         _, input_name = input_name.split("//", 2)
-        logger = log.Logs().get_logger("{}_input".format(APP_NAME))
-
-        # Log level configuration
-        log_level = get_log_level(session_key, logger)
-        logger.setLevel(log_level)
+        logger = get_logger(session_key, "input")
 
         logger.debug("Modular input invoked.")