Merge pull request #13 from scality/feature/RELENG-6059-extra-label

RELENG-6059 add the ability to add more relabel jobs

Author: tcarmet

charts/gh-action-exporter/templates/config.yaml

@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ .Release.Name }}-config
+data:
+  config.yaml: |-
+{{ .Values.config | toYaml | indent 4 }}

charts/gh-action-exporter/templates/deployment.yaml

@@ -15,6 +15,7 @@ spec:
     metadata:
       {{- with .Values.podAnnotations }}
       annotations:
+        checksum/config: {{ include (print $.Template.BasePath "/config.yaml") . | sha256sum }}
         {{- toYaml . | nindent 8 }}
       {{- end }}
       labels:
@@ -47,6 +48,16 @@ spec:
               port: http
           resources:
             {{- toYaml .Values.resources | nindent 12 }}
+          env:
+            - name: CONFIG_FILE
+              value: /app/config/config.yaml
+          volumeMounts:
+            - name: config
+              mountPath: /app/config
+      volumes:
+        - name: config
+          configMap:
+            name: {{ .Release.Name }}-config
       {{- with .Values.nodeSelector }}
       nodeSelector:
         {{- toYaml . | nindent 8 }}

charts/gh-action-exporter/values.yaml

@@ -14,6 +14,10 @@ imagePullSecrets: []
 nameOverride: ""
 fullnameOverride: ""
 
+
+config:
+  job_relabelling: []
+
 serviceAccount:
   # Specifies whether a service account should be created
   create: true

gh_actions_exporter/Webhook.py

@@ -1,4 +1,5 @@
 import logging
+from gh_actions_exporter.config import Settings
 from gh_actions_exporter.types import WebHook
 from gh_actions_exporter.metrics import Metrics
 
@@ -9,10 +10,11 @@ class WebhookManager(object):
     event: str
     payload: WebHook
 
-    def __init__(self, payload: WebHook, event: str, metrics: Metrics):
+    def __init__(self, payload: WebHook, event: str, metrics: Metrics, settings: Settings):
         self.event = event
         self.payload = payload
         self.metrics = metrics
+        self.settings = settings
 
     def __call__(self, *args, **kwargs):
         # Check if we managed this event
@@ -28,8 +30,8 @@ def workflow_run(self):
         self.metrics.handle_workflow_rebuild(self.payload)
 
     def workflow_job(self):
-        self.metrics.handle_job_status(self.payload)
-        self.metrics.handle_job_duration(self.payload)
+        self.metrics.handle_job_status(self.payload, self.settings)
+        self.metrics.handle_job_duration(self.payload, self.settings)
 
     def ping(self):
         logger.info('Ping from Github')

gh_actions_exporter/config.py

@@ -0,0 +1,55 @@
+import yaml
+from enum import Enum
+from pathlib import Path
+from typing import Dict, Any, List, Optional
+
+from pydantic import BaseModel, BaseSettings
+
+
+def yaml_config_settings_source(settings: BaseSettings) -> Dict[str, Any]:
+    """
+    A simple settings source that loads variables from a yaml file
+
+    """
+    config_file: Path = settings.__config__.config.config_file
+    if config_file:
+        return yaml.full_load(config_file.read_text())
+    return {}
+
+
+class RelabelType(str, Enum):
+    name = 'name'
+    label = 'label'
+
+
+class Relabel(BaseModel):
+    label: str
+    type: RelabelType = RelabelType.label
+    description: Optional[str]
+    default: Optional[str] = None
+    values: List[str]
+
+
+class ConfigFile(BaseSettings):
+    config_file: Optional[Path]
+
+
+class Settings(BaseSettings):
+    job_relabelling: Optional[List[Relabel]] = []
+
+    class Config:
+        config: ConfigFile = ConfigFile()
+
+        @classmethod
+        def customise_sources(
+            cls,
+            init_settings,
+            env_settings,
+            file_secret_settings,
+        ):
+            return (
+                init_settings,
+                yaml_config_settings_source,
+                env_settings,
+                file_secret_settings,
+            )

gh_actions_exporter/main.py

@@ -1,13 +1,26 @@
 import uvicorn
 
-from fastapi import FastAPI, Request
+from fastapi import Depends, FastAPI, Request
+from functools import lru_cache
 
 from gh_actions_exporter.metrics import prometheus_metrics, Metrics
 from gh_actions_exporter.types import WebHook
 from gh_actions_exporter.Webhook import WebhookManager
+from gh_actions_exporter.config import Settings
+
+
+@lru_cache()
+def get_settings() -> Settings:
+    return Settings()
+
+
+@lru_cache()
+def metrics() -> Metrics:
+    return Metrics(get_settings())
+
 
 app = FastAPI()
-metrics = Metrics()
+
 
 app.add_route('/metrics', prometheus_metrics)
 
@@ -18,13 +31,23 @@ def index():
 
 
 @app.post("/webhook", status_code=202)
-async def webhook(webhook: WebHook, request: Request):
-    WebhookManager(payload=webhook, event=request.headers['X-Github-Event'], metrics=metrics)()
+async def webhook(
+    webhook: WebHook,
+    request: Request,
+    settings: Settings = Depends(get_settings),
+    metrics: Metrics = Depends(metrics)
+):
+    WebhookManager(
+        payload=webhook,
+        event=request.headers['X-Github-Event'],
+        metrics=metrics,
+        settings=settings
+    )()
     return "Accepted"
 
 
 @app.delete("/clear", status_code=200)
-async def clear():
+async def clear(metrics: Metrics = Depends(metrics)):
     metrics.workflow_rebuild.clear()
     metrics.workflow_duration.clear()
     metrics.job_duration.clear()

gh_actions_exporter/metrics.py

@@ -3,9 +3,11 @@
 from prometheus_client import (CONTENT_TYPE_LATEST, REGISTRY,
                                CollectorRegistry, generate_latest)
 from prometheus_client.multiprocess import MultiProcessCollector
+from typing import Dict, List
 from fastapi.requests import Request
 from fastapi.responses import Response
-from gh_actions_exporter.types import WebHook
+from gh_actions_exporter.config import Relabel, RelabelType, Settings
+from gh_actions_exporter.types import WebHook, WorkflowJob
 from prometheus_client import Counter, Histogram
 
 
@@ -23,8 +25,8 @@ def prometheus_metrics(request: Request) -> Response:
 
 
 class Metrics(object):
-
-    def __init__(self):
+    def __init__(self, settings: Settings):
+        self.settings = settings
         self.workflow_labelnames = [
             'repository',
             'workflow_name',
@@ -36,6 +38,9 @@ def __init__(self):
             'repository_visibility',
             'runner_type'
         ]
+        for relabel in self.settings.job_relabelling:
+            self.job_labelnames.append(relabel.label)
+
         self.workflow_rebuild = Counter(
             'github_actions_workflow_rebuild_count', 'The number of workflow rebuild',
             labelnames=self.workflow_labelnames
@@ -123,13 +128,38 @@ def runner_type(self, webhook: WebHook) -> str:
             return 'self-hosted'
         return 'github-hosted'
 
-    def job_labels(self, webhook: WebHook) -> dict:
+    def relabel_job_labels(self, relabel: Relabel, labels: List[str]) -> Dict[str, str or None]:
+        result = {
+            relabel.label: relabel.default
+        }
+        for label in relabel.values:
+            if label in labels:
+                result[relabel.label] = label
+        return result
+
+    def relabel_job_names(self, relabel: Relabel, job: WorkflowJob) -> dict:
+        if job.status == 'queued':
+            return dict()
+        result = {
+            relabel.label: relabel.default
+        }
+        for label in relabel.values:
+            if label in job.runner_name:
+                result[relabel.label] = label
+        return result
+
+    def job_labels(self, webhook: WebHook, settings: Settings) -> dict:
         labels = dict(
             runner_type=self.runner_type(webhook),
             job_name=webhook.workflow_job.name,
             repository_visibility=webhook.repository.visibility,
             repository=webhook.repository.full_name,
         )
+        for relabel in settings.job_relabelling:
+            if relabel.type == RelabelType.label:
+                labels.update(self.relabel_job_labels(relabel, webhook.workflow_job.labels))
+            elif relabel.type == RelabelType.name:
+                labels.update(self.relabel_job_names(relabel, webhook.workflow_job))
         return labels
 
     def handle_workflow_rebuild(self, webhook: WebHook):
@@ -165,8 +195,8 @@ def handle_workflow_duration(self, webhook: WebHook):
                         - webhook.workflow_run.run_started_at.timestamp())
             self.workflow_duration.labels(**labels).observe(duration)
 
-    def handle_job_status(self, webhook: WebHook):
-        labels = self.job_labels(webhook)
+    def handle_job_status(self, webhook: WebHook, settings: Settings):
+        labels = self.job_labels(webhook, settings)
         if webhook.workflow_job.conclusion:
             if webhook.workflow_job.conclusion == 'success':
                 self.job_status_success.labels(**labels).inc()
@@ -180,8 +210,8 @@ def handle_job_status(self, webhook: WebHook):
         elif webhook.workflow_job.status == 'queued':
             self.job_status_queued.labels(**labels).inc()
 
-    def handle_job_duration(self, webhook: WebHook):
-        labels = self.job_labels(webhook)
+    def handle_job_duration(self, webhook: WebHook, settings: Settings):
+        labels = self.job_labels(webhook, settings)
         if webhook.workflow_job.conclusion:
             duration = (webhook.workflow_job.completed_at.timestamp()
                         - webhook.workflow_job.started_at.timestamp())

gh_actions_exporter/types.py

@@ -18,6 +18,7 @@ class WorkflowJob(BaseModel):
     run_id: int
     run_url: str
     run_attempt: int
+    runner_name: Optional[str] = None
     node_id: str
     head_sha: str
     url: str