feature(argus): call client with extra headers

since argus serivce will need more headers for it's to be operational,
we'll need to change the calls to it, to pass the needed headers.

(cherry picked from commit 6e4e1fb)

Author: fruch

argus/client/base.py

@@ -34,15 +34,20 @@ class Routes():
         FETCH_RESULTS = "/testrun/$type/$id/fetch_results"
         FINALIZE = "/testrun/$type/$id/finalize"
 
-    def __init__(self, auth_token: str, base_url: str, api_version="v1") -> None:
+    def __init__(self, auth_token: str, base_url: str, api_version="v1", extra_headers: dict | None = None) -> None:
         self._auth_token = auth_token
         self._base_url = base_url
         self._api_ver = api_version
+        self._extra_headers = extra_headers or {}
 
     @property
     def auth_token(self) -> str:
         return self._auth_token
 
+    @property
+    def extra_headers(self) -> dict:
+        return self._extra_headers
+
     def verify_location_params(self, endpoint: str, location_params: dict[str, str]) -> bool:
         required_params: list[str] = re.findall(r"\$[\w_]+", endpoint)
         for param in required_params:
@@ -88,6 +93,7 @@ def request_headers(self):
             "Authorization": f"token {self.auth_token}",
             "Accept": "application/json",
             "Content-Type": "application/json",
+            **self.extra_headers,
         }
 
     def get(self, endpoint: str, location_params: dict[str, str] = None, params: dict = None) -> requests.Response:

argus/client/driver_matrix_tests/cli.py

@@ -6,6 +6,7 @@
 from argus.common.enums import TestStatus
 
 from argus.client.driver_matrix_tests.client import ArgusDriverMatrixClient
+from argus.client.generic.cli import validate_extra_headers
 
 LOGGER = logging.getLogger(__name__)
 
@@ -15,86 +16,92 @@ def cli():
     pass
 
 
-def _submit_driver_result_internal(api_key: str, base_url: str, run_id: str, metadata_path: str):
+def _submit_driver_result_internal(api_key: str, base_url: str, run_id: str, metadata_path: str, extra_headers: dict):
     metadata = json.loads(Path(metadata_path).read_text(encoding="utf-8"))
     LOGGER.info("Submitting results for %s [%s/%s] to Argus...", run_id, metadata["driver_name"], metadata["driver_type"])
     raw_xml = (Path(metadata_path).parent / metadata["junit_result"]).read_bytes()
-    client = ArgusDriverMatrixClient(run_id=run_id, auth_token=api_key, base_url=base_url)
+    client = ArgusDriverMatrixClient(run_id=run_id, auth_token=api_key, base_url=base_url, extra_headers=extra_headers)
     client.submit_driver_result(driver_name=metadata["driver_name"], driver_type=metadata["driver_type"], raw_junit_data=base64.encodebytes(raw_xml))
     LOGGER.info("Done.")
 
-def _submit_driver_failure_internal(api_key: str, base_url: str, run_id: str, metadata_path: str):
+def _submit_driver_failure_internal(api_key: str, base_url: str, run_id: str, metadata_path: str, extra_headers: dict):
     metadata = json.loads(Path(metadata_path).read_text(encoding="utf-8"))
     LOGGER.info("Submitting failure for %s [%s/%s] to Argus...", run_id, metadata["driver_name"], metadata["driver_type"])
-    client = ArgusDriverMatrixClient(run_id=run_id, auth_token=api_key, base_url=base_url)
+    client = ArgusDriverMatrixClient(run_id=run_id, auth_token=api_key, base_url=base_url, extra_headers=extra_headers)
     client.submit_driver_failure(driver_name=metadata["driver_name"], driver_type=metadata["driver_type"], failure_reason=metadata["failure_reason"])
     LOGGER.info("Done.")
 
 
 @click.command("submit-run")
-@click.option("--api-key", help="Argus API key for authorization", required=True)
+@click.option("--api-key", help="Argus API key for authorization", required=True, envvar='ARGUS_AUTH_TOKEN')
+@click.option("--extra-headers", default={}, type=click.UNPROCESSED, callback=validate_extra_headers, help="extra headers to pass to argus, should be in json format", envvar='ARGUS_EXTRA_HEADERS')
 @click.option("--base-url", default="https://argus.scylladb.com", help="Base URL for argus instance")
 @click.option("--id", "run_id", required=True, help="UUID (v4 or v1) unique to the job")
 @click.option("--build-id", required=True, help="Unique job identifier in the build system, e.g. scylla-master/group/job for jenkins (The full path)")
 @click.option("--build-url", required=True, help="Job URL in the build system")
-def submit_driver_matrix_run(api_key: str, base_url: str, run_id: str, build_id: str, build_url: str):
+def submit_driver_matrix_run(api_key: str, base_url: str, run_id: str, build_id: str, build_url: str, extra_headers: dict):
     LOGGER.info("Submitting %s (%s) to Argus...", build_id, run_id)
-    client = ArgusDriverMatrixClient(run_id=run_id, auth_token=api_key, base_url=base_url)
+    client = ArgusDriverMatrixClient(run_id=run_id, auth_token=api_key, base_url=base_url, extra_headers=extra_headers)
     client.submit_driver_matrix_run(job_name=build_id, job_url=build_url)
     LOGGER.info("Done.")
 
 
 @click.command("submit-driver")
-@click.option("--api-key", help="Argus API key for authorization", required=True)
+@click.option("--api-key", help="Argus API key for authorization", required=True, envvar='ARGUS_AUTH_TOKEN')
+@click.option("--extra-headers", default={}, type=click.UNPROCESSED, callback=validate_extra_headers, help="extra headers to pass to argus, should be in json format", envvar='ARGUS_EXTRA_HEADERS')
 @click.option("--base-url", default="https://argus.scylladb.com", help="Base URL for argus instance")
 @click.option("--id", "run_id", required=True, help="UUID (v4 or v1) unique to the job")
 @click.option("--metadata-path", required=True, help="Path to the metadata .json file that contains path to junit xml and other required information")
-def submit_driver_result(api_key: str, base_url: str, run_id: str, metadata_path: str):
-    _submit_driver_result_internal(api_key=api_key, base_url=base_url, run_id=run_id, metadata_path=metadata_path)
+def submit_driver_result(api_key: str, base_url: str, run_id: str, metadata_path: str, extra_headers: dict):
+    _submit_driver_result_internal(api_key=api_key, base_url=base_url, run_id=run_id, metadata_path=metadata_path, extra_headers=extra_headers)
 
 
 @click.command("fail-driver")
-@click.option("--api-key", help="Argus API key for authorization", required=True)
+@click.option("--api-key", help="Argus API key for authorization", required=True, envvar='ARGUS_AUTH_TOKEN')
+@click.option("--extra-headers", default={}, type=click.UNPROCESSED, callback=validate_extra_headers, help="extra headers to pass to argus, should be in json format", envvar='ARGUS_EXTRA_HEADERS')
 @click.option("--base-url", default="https://argus.scylladb.com", help="Base URL for argus instance")
 @click.option("--id", "run_id", required=True, help="UUID (v4 or v1) unique to the job")
 @click.option("--metadata-path", required=True, help="Path to the metadata .json file that contains path to junit xml and other required information")
-def submit_driver_failure(api_key: str, base_url: str, run_id: str, metadata_path: str):
-    _submit_driver_failure_internal(api_key=api_key, base_url=base_url, run_id=run_id, metadata_path=metadata_path)
+def submit_driver_failure(api_key: str, base_url: str, run_id: str, metadata_path: str, extra_headers: dict):
+    _submit_driver_failure_internal(api_key=api_key, base_url=base_url, run_id=run_id, metadata_path=metadata_path, extra_headers=extra_headers)
 
 
 @click.command("submit-or-fail-driver")
-@click.option("--api-key", help="Argus API key for authorization", required=True)
+@click.option("--api-key", help="Argus API key for authorization", required=True, envvar='ARGUS_AUTH_TOKEN')
+@click.option("--extra-headers", default={}, type=click.UNPROCESSED, callback=validate_extra_headers, help="extra headers to pass to argus, should be in json format", envvar='ARGUS_EXTRA_HEADERS')
 @click.option("--base-url", default="https://argus.scylladb.com", help="Base URL for argus instance")
 @click.option("--id", "run_id", required=True, help="UUID (v4 or v1) unique to the job")
 @click.option("--metadata-path", required=True, help="Path to the metadata .json file that contains path to junit xml and other required information")
-def submit_or_fail_driver(api_key: str, base_url: str, run_id: str, metadata_path: str):
+def submit_or_fail_driver(api_key: str, base_url: str, run_id: str, metadata_path: str, extra_headers: dict):
     metadata = json.loads(Path(metadata_path).read_text(encoding="utf-8"))
     if metadata.get("failure_reason"):
-        _submit_driver_failure_internal(api_key=api_key, base_url=base_url, run_id=run_id, metadata_path=metadata_path)
+        _submit_driver_failure_internal(api_key=api_key, base_url=base_url, run_id=run_id, metadata_path=metadata_path, extra_headers=extra_headers)
     else:
-        _submit_driver_result_internal(api_key=api_key, base_url=base_url, run_id=run_id, metadata_path=metadata_path)
+        _submit_driver_result_internal(api_key=api_key, base_url=base_url, run_id=run_id, metadata_path=metadata_path, extra_headers=extra_headers)
 
 
 @click.command("submit-env")
-@click.option("--api-key", help="Argus API key for authorization", required=True)
+@click.option("--api-key", help="Argus API key for authorization", required=True, envvar='ARGUS_AUTH_TOKEN')
+@click.option("--extra-headers", default={}, type=click.UNPROCESSED, callback=validate_extra_headers, help="extra headers to pass to argus, should be in json format", envvar='ARGUS_EXTRA_HEADERS')
 @click.option("--base-url", default="https://argus.scylladb.com", help="Base URL for argus instance")
 @click.option("--id", "run_id", required=True, help="UUID (v4 or v1) unique to the job")
 @click.option("--env-path", required=True, help="Path to the Build-00.txt file that contains environment information about Scylla")
-def submit_driver_env(api_key: str, base_url: str, run_id: str, env_path: str):
+def submit_driver_env(api_key: str, base_url: str, run_id: str, env_path: str, extra_headers: dict):
     LOGGER.info("Submitting environment for run %s to Argus...", run_id)
     raw_env = Path(env_path).read_text()
-    client = ArgusDriverMatrixClient(run_id=run_id, auth_token=api_key, base_url=base_url)
+    client = ArgusDriverMatrixClient(run_id=run_id, auth_token=api_key, base_url=base_url, extra_headers=extra_headers)
     client.submit_env(raw_env)
     LOGGER.info("Done.")
 
 
 @click.command("finish-run")
-@click.option("--api-key", help="Argus API key for authorization", required=True)
+@click.option("--api-key", help="Argus API key for authorization", required=True, envvar='ARGUS_AUTH_TOKEN')
+@click.option("--extra-headers", default={}, type=click.UNPROCESSED, callback=validate_extra_headers, help="extra headers to pass to argus, should be in json format", envvar='ARGUS_EXTRA_HEADERS')
 @click.option("--base-url", default="https://argus.scylladb.com", help="Base URL for argus instance")
 @click.option("--id", "run_id", required=True, help="UUID (v4 or v1) unique to the job")
 @click.option("--status", required=True, help="Resulting job status")
-def finish_driver_matrix_run(api_key: str, base_url: str, run_id: str, status: str):
-    client = ArgusDriverMatrixClient(run_id=run_id, auth_token=api_key, base_url=base_url)
+def finish_driver_matrix_run(api_key: str, base_url: str, run_id: str, status: str, extra_headers: dict):
+    client = ArgusDriverMatrixClient(run_id=run_id, auth_token=api_key, base_url=base_url, extra_headers=extra_headers)
     client.finalize_run(run_type=ArgusDriverMatrixClient.test_type, run_id=run_id, body={"status": TestStatus(status)})
 
 

argus/client/driver_matrix_tests/client.py

@@ -12,8 +12,8 @@ class Routes(ArgusClient.Routes):
         SUBMIT_DRIVER_FAILURE = "/driver_matrix/result/fail"
         SUBMIT_ENV = "/driver_matrix/env/submit"
 
-    def __init__(self, run_id: UUID, auth_token: str, base_url: str, api_version="v1") -> None:
-        super().__init__(auth_token, base_url, api_version)
+    def __init__(self, run_id: UUID, auth_token: str, base_url: str, api_version="v1", extra_headers: dict | None = None) -> None:
+        super().__init__(auth_token, base_url, api_version, extra_headers=extra_headers)
         self.run_id = run_id
 
     def submit_driver_matrix_run(self, job_name: str, job_url: str) -> None:

argus/client/generic/cli.py

@@ -1,3 +1,12 @@
+<<<<<<< HEAD
+||||||| parent of 6e4e1fbc0 (feature(argus): call client with extra headers)
+import json
+from pathlib import Path
+=======
+import json
+from json.decoder import JSONDecodeError
+from pathlib import Path
+>>>>>>> 6e4e1fbc0 (feature(argus): call client with extra headers)
 import click
 import logging
 
@@ -6,39 +15,68 @@
 
 LOGGER = logging.getLogger(__name__)
 
+def validate_extra_headers(ctx, param, value):
+    if isinstance(value, dict):
+        return value
+
+    try:
+        return json.loads(value)
+    except JSONDecodeError as ex:
+        raise click.BadParameter(f"--extra-headers should be in json format:\n\n{value}\n\n{ex}")
+
 
 @click.group
 def cli():
     pass
 
 
 @click.command("submit")
-@click.option("--api-key", help="Argus API key for authorization", required=True)
+@click.option("--api-key", help="Argus API key for authorization", required=True, envvar='ARGUS_AUTH_TOKEN')
 @click.option("--base-url", default="https://argus.scylladb.com", help="Base URL for argus instance")
 @click.option("--id", required=True, help="UUID (v4 or v1) unique to the job")
 @click.option("--build-id", required=True, help="Unique job identifier in the build system, e.g. scylla-master/group/job for jenkins (The full path)")
 @click.option("--build-url", required=True, help="Job URL in the build system")
 @click.option("--started-by", required=True, help="Username of the user who started the job")
 @click.option("--scylla-version", required=False, default=None, help="Version of Scylla used for this job")
-def submit_run(api_key: str, base_url: str, id: str, build_id: str, build_url: str, started_by: str, scylla_version: str = None):
+@click.option("--extra-headers", default={}, type=click.UNPROCESSED, callback=validate_extra_headers, help="extra headers to pass to argus, should be in json format", envvar='ARGUS_EXTRA_HEADERS')
+def submit_run(api_key: str, base_url: str, id: str, build_id: str, build_url: str, started_by: str, scylla_version: str = None, extra_headers: dict | None = None):
     LOGGER.info("Submitting %s (%s) to Argus...", build_id, id)
-    client = ArgusGenericClient(auth_token=api_key, base_url=base_url)
+    client = ArgusGenericClient(auth_token=api_key, base_url=base_url, extra_headers=extra_headers)
     client.submit_generic_run(build_id=build_id, run_id=id, started_by=started_by, build_url=build_url, scylla_version=scylla_version)
     LOGGER.info("Done.")
 
 
 @click.command("finish")
-@click.option("--api-key", help="Argus API key for authorization", required=True)
+@click.option("--api-key", help="Argus API key for authorization", required=True, envvar='ARGUS_AUTH_TOKEN')
 @click.option("--base-url", default="https://argus.scylladb.com", help="Base URL for argus instance")
 @click.option("--id", required=True, help="UUID (v4 or v1) unique to the job")
 @click.option("--status", required=True, help="Resulting job status")
 @click.option("--scylla-version", required=False, default=None, help="Version of Scylla used for this job")
-def finish_run(api_key: str, base_url: str, id: str, status: str, scylla_version: str = None):
-    client = ArgusGenericClient(auth_token=api_key, base_url=base_url)
+@click.option("--extra-headers", default={}, type=click.UNPROCESSED, callback=validate_extra_headers, help="extra headers to pass to argus, should be in json format", envvar='ARGUS_EXTRA_HEADERS')
+def finish_run(api_key: str, base_url: str, id: str, status: str, scylla_version: str = None, extra_headers: dict | None = None):
+    client = ArgusGenericClient(auth_token=api_key, base_url=base_url, extra_headers=extra_headers)
     status = TestStatus(status)
     client.finalize_generic_run(run_id=id, status=status, scylla_version=scylla_version)
 
 
+@click.command("trigger-jobs")
+@click.option("--api-key", help="Argus API key for authorization", required=True, envvar='ARGUS_AUTH_TOKEN')
+@click.option("--base-url", default="https://argus.scylladb.com", help="Base URL for argus instance")
+@click.option("--version", help="Scylla version to filter plans by", default=None, required=False)
+@click.option("--plan-id", help="Specific plan id for filtering", default=None, required=False)
+@click.option("--release", help="Release name to filter plans by", default=None, required=False)
+@click.option("--job-info-file", required=True, help="JSON file with trigger information (see detailed docs)")
+@click.option("--extra-headers", default={}, type=click.UNPROCESSED, callback=validate_extra_headers, help="extra headers to pass to argus, should be in json format", envvar='ARGUS_EXTRA_HEADERS')
+def trigger_jobs(api_key: str, base_url: str, job_info_file: str, version: str, plan_id: str, release: str, extra_headers: dict | None = None):
+    client = ArgusGenericClient(auth_token=api_key, base_url=base_url, extra_headers=extra_headers)
+    path = Path(job_info_file)
+    if not path.exists():
+        LOGGER.error("File not found: %s", job_info_file)
+        exit(128)
+    payload = json.load(path.open("rt", encoding="utf-8"))
+    client.trigger_jobs({ "release": release, "version": version, "plan_id": plan_id, **payload })
+
+
 cli.add_command(submit_run)
 cli.add_command(finish_run)
 

argus/client/generic/client.py

@@ -7,8 +7,12 @@
 class ArgusGenericClient(ArgusClient):
     test_type = "generic"
     schema_version: None = "v1"
-    def __init__(self, auth_token: str, base_url: str, api_version="v1") -> None:
-        super().__init__(auth_token, base_url, api_version)
+
+    class Routes(ArgusClient.Routes):
+        TRIGGER_JOBS = "/planning/plan/trigger"
+
+    def __init__(self, auth_token: str, base_url: str, api_version="v1", extra_headers: dict | None = None) -> None:
+        super().__init__(auth_token, base_url, api_version, extra_headers=extra_headers)
 
     def submit_generic_run(self, build_id: str, run_id: str, started_by: str, build_url: str, scylla_version: str | None = None):
         request_body = {
@@ -28,4 +32,4 @@ def finalize_generic_run(self, run_id: str, status: str, scylla_version: str | N
             "status": status,
             "scylla_version": scylla_version,
         })
-        self.check_response(response)
+        self.check_response(response)

argus/client/sct/client.py

@@ -27,8 +27,8 @@ class Routes(ArgusClient.Routes):
         SUBMIT_EVENTS = "/sct/$id/events/submit"
         SUBMIT_JUNIT_REPORT = "/sct/$id/junit/submit"
 
-    def __init__(self, run_id: UUID, auth_token: str, base_url: str, api_version="v1") -> None:
-        super().__init__(auth_token, base_url, api_version)
+    def __init__(self, run_id: UUID, auth_token: str, base_url: str, api_version="v1", extra_headers: dict | None = None) -> None:
+        super().__init__(auth_token, base_url, api_version, extra_headers=extra_headers)
         self.run_id = run_id
 
     def submit_sct_run(self, job_name: str, job_url: str, started_by: str, commit_id: str,

argus/client/sirenada/client.py

@@ -43,9 +43,9 @@ class ArgusSirenadaClient(ArgusClient):
         "skipped": "skipped"
     }
 
-    def __init__(self, auth_token: str, base_url: str, api_version="v1") -> None:
+    def __init__(self, auth_token: str, base_url: str, api_version="v1", extra_headers: dict | None = None) -> None:
         self.results_path: Path | None = None
-        super().__init__(auth_token, base_url, api_version)
+        super().__init__(auth_token, base_url, api_version, extra_headers=extra_headers)
 
     def _verify_required_files_exist(self, results_path: Path):
         assert (results_path / self._junit_xml_filename).exists(), "Missing jUnit XML results file!"

argus/version

@@ -1 +1 @@
-0.13.0
+0.14.2

sdcm/utils/argus.py

@@ -40,7 +40,8 @@ def get_argus_client(run_id: UUID | str) -> ArgusSCTClient:
     if not is_uuid(run_id):
         raise ArgusError("Malformed UUID provided")
     creds = KeyStore().get_argus_rest_credentials()
-    argus_client = ArgusSCTClient(run_id=run_id, auth_token=creds["token"], base_url=creds["baseUrl"])
+    argus_client = ArgusSCTClient(
+        run_id=run_id, auth_token=creds["token"], base_url=creds["baseUrl"], extra_headers=creds.get("extra_headers"))
 
     return argus_client