Support query parameters for Exporters

We have supported query parameters for the Exporter's endpoint. Users can scrape the endpoint
with specific query parameters by adding their query parameters to the "path" field when
creating a new Exporter in Promgen. Currently, the list of values for parameters are not supported.

For example:
```
path = /metrics?match[]={job="prometheus"}&param1=foo&param2=bar
```

Author: hoangpn

promgen/forms.py

@@ -109,6 +109,32 @@ class Meta:
             "scheme": forms.Select(attrs={"class": "form-control"}),
         }
 
+    def clean_path(self):
+        from urllib.parse import parse_qs, urlparse
+
+        path = self.cleaned_data["path"]
+        if not path:
+            return path
+
+        # Validate path and query parameters
+        parsed = urlparse(path)
+        query_params = parse_qs(qs=parsed.query)
+        query_params = {k: list(set(v)) for k, v in query_params.items()}
+        for key, value in query_params.items():
+            validators.validate_utf8(key)
+            if len(value) > 1:
+                raise ValidationError(
+                    "List values are currently not supported for query parameters."
+                )
+            for v in value:
+                validators.validate_utf8(v)
+
+        # Reconstruct the path with cleaned params
+        query_string = "&".join(
+            f"{key}={value}" for key, values in query_params.items() for value in values
+        )
+        return parsed.path + ("?" + query_string if query_string else "")
+
 
 class ServiceRegister(forms.ModelForm):
     class Meta:

promgen/prometheus.py

@@ -6,7 +6,7 @@
 import logging
 import subprocess
 import tempfile
-from urllib.parse import urljoin
+from urllib.parse import parse_qs, urljoin, urlparse
 
 import yaml
 from dateutil import parser
@@ -133,15 +133,20 @@ def render_config(service=None, project=None, services=None, projects=None, farm
             "job": exporter.job,
             "__scheme__": exporter.scheme,
         }
+
         if exporter.path:
-            labels["__metrics_path__"] = exporter.path
+            parsed = urlparse(exporter.path)
+            labels["__metrics_path__"] = parsed.path
+            query_params = parse_qs(parsed.query)
+            for key, value in query_params.items():
+                labels[f"__param_{key}"] = value[0]
 
         hosts = []
         if farms is None or exporter.project.farm in farms:
             for host in exporter.project.farm.host_set.all():
                 hosts.append(f"{host.name}:{exporter.port}")
 
-        data.append({"labels": labels, "targets": hosts})
+            data.append({"labels": labels, "targets": hosts})
     return json.dumps(data, indent=2, sort_keys=True)
 
 

promgen/tests/examples/export.targets_with_query_params.json

@@ -0,0 +1,20 @@
+[
+  {
+    "labels": {
+      "__farm_source": "promgen",
+      "__scheme__": "http",
+      "__shard": "test-shard",
+      "farm": "test-farm",
+      "job": "node",
+      "project": "test-project",
+      "service": "test-service",
+      "__metrics_path__": "/metrics",
+      "__param_match[]": "{job=\"prometheus\"}",
+      "__param_param1": "foo",
+      "__param_param2": "bar"
+    },
+    "targets": [
+      "example.com:9100"
+    ]
+  }
+]

promgen/tests/test_renderers.py

@@ -4,7 +4,7 @@
 from django.urls import reverse
 from yaml import safe_load
 
-from promgen import tests
+from promgen import models, tests
 
 
 class RendererTests(tests.PromgenTest):
@@ -32,3 +32,15 @@ def test_global_urls(self):
         response = self.client.get(reverse("api:all-urls"))
         self.assertEqual(response.status_code, 200)
         self.assertEqual(response.json(), expected)
+
+    def test_targets_with_query_params(self):
+        exporter = models.Exporter.objects.get(pk=1)
+        exporter.path = (
+            '/metrics?match[]={job="prometheus"}&param1=foo&param2=bar'
+        )
+        exporter.save()
+
+        expected = tests.Data("examples", "export.targets_with_query_params.json").json()
+        response = self.client.get(reverse("api:all-targets"))
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(response.json(), expected)

promgen/tests/test_web.py

@@ -1,5 +1,7 @@
 # Copyright (c) 2022 LINE Corporation
 # These sources are released under the terms of the MIT license: see LICENSE
+from unittest import mock
+
 from django.urls import reverse
 from guardian.shortcuts import assign_perm, remove_perm
 
@@ -285,3 +287,71 @@ def test_merge_users_no_permissions(self):
             reverse("admin:login") + "?next=" + reverse("admin:user-merge"),
             "Non-admin user redirected to login page",
         )
+
+    @mock.patch("promgen.models.Audit.log")
+    @mock.patch("promgen.signals.trigger_write_config.send")
+    def test_register_exporter(self, log_mock, write_mock):
+        self.force_login(username="demo")
+        assign_perm("project_editor", self.user, models.Project.objects.get(pk=1))
+
+        # Check creating an exporter with just the required fields and no path
+        response = self.client.post(
+            reverse("project-exporter", kwargs={"pk": 1}),
+            {"job": "node", "port": 9101, "scheme": "http"},
+        )
+        self.assertEqual(
+            response.status_code, 302, msg="Exporter created successfully without path"
+        )
+
+        # Check successfully creating an exporter with query parameters in the path
+        response = self.client.post(
+            reverse("project-exporter", kwargs={"pk": 1}),
+            {
+                "job": "node",
+                "port": 9101,
+                "scheme": "http",
+                "path": "/metrics?param1=foo&param2=bar",
+            },
+        )
+        self.assertEqual(response.status_code, 302)
+        self.assertTrue(
+            models.Exporter.objects.filter(
+                path="/metrics?param1=foo&param2=bar",
+            ).exists(),
+            "Exporter with query parameters in path created successfully",
+        )
+
+        # Check creating an exporter with array query parameter should not be allowed
+        response = self.client.post(
+            reverse("project-exporter", kwargs={"pk": 1}),
+            {
+                "job": "node",
+                "port": 9101,
+                "scheme": "http",
+                "path": "/metrics?list[]=foo&list[]=bar",
+            },
+        )
+        self.assertEqual(response.status_code, 200, msg="Exporter created failed")
+        self.assertEqual(
+            response.context_data["form"].errors,
+            {"path": ["List values are currently not supported for query parameters."]},
+            msg="Exporter with array query parameters should not be allowed",
+        )
+
+        # Check creating an exporter with redundant query parameters should be cleaned up
+        response = self.client.post(
+            reverse("project-exporter", kwargs={"pk": 1}),
+            {
+                "job": "node",
+                "port": 9101,
+                "scheme": "http",
+                "path": "/metrics?param=value&param&param=",
+            },
+        )
+        self.assertEqual(response.status_code, 302)
+        self.assertTrue(
+            models.Exporter.objects.filter(
+                path="/metrics?param=value",
+            ).exists(),
+            "Redundant query parameters should be cleaned up and exporter created successfully",
+        )

promgen/validators.py

@@ -65,3 +65,10 @@ def datetime(value):
         parser.parse(value)
     except ValueError:
         raise ValidationError("Invalid timestamp")
+
+
+def validate_utf8(value):
+    try:
+        value.encode("utf-8").decode("utf-8")
+    except (UnicodeEncodeError, UnicodeDecodeError):
+        raise ValidationError("Invalid UTF-8 string.")