Validate Prometheus Connection (#118)

This falls under the category of #61. 

Just trying to ensure the user is aware of the error without an
explosion of irrelevant stack traces.

Author: bh2smith

src/main.py

@@ -26,6 +26,7 @@
 from src.args import Args
 from src.config import RuntimeConfig
 from src.job import Job
+from src.logger import log
 
 
 async def main(jobs: list[Job]) -> None:
@@ -37,7 +38,10 @@ async def main(jobs: list[Job]) -> None:
     """
     tasks = [job.run() for job in jobs]
     for completed_task in asyncio.as_completed(tasks):
-        await completed_task
+        try:
+            await completed_task
+        except Exception as e:  # pylint: disable=broad-exception-caught
+            log.error("Error in job execution: %s", str(e))
 
 
 if __name__ == "__main__":

src/metrics.py

@@ -6,6 +6,7 @@
 from time import perf_counter
 from typing import Any
 
+import requests
 from prometheus_client import CollectorRegistry, Counter, Gauge, push_to_gateway
 
 # MARKER: pylint-bug
@@ -15,6 +16,8 @@
 from src.interfaces import Named
 from src.logger import log
 
+SUCCESS_STATUS = 200
+
 
 def log_job_metrics(prometheus_url: str, job_metrics: dict[str, Any]) -> None:
     """Log metrics about a job to a prometheus pushgateway."""
@@ -60,6 +63,7 @@ async def wrapper(
         if not (prometheus_url := env("PROMETHEUS_PUSHGATEWAY_URL")):
             return await func(self, *args, **kwargs)
 
+        validate_prometheus_url(prometheus_url)
         run_id = uuid.uuid4().hex
         start = perf_counter()
         success = False
@@ -82,3 +86,21 @@ async def wrapper(
             log_job_metrics(prometheus_url, metrics)
 
     return wrapper
+
+
+def validate_prometheus_url(prometheus_url: str) -> None:
+    """Ensure Valid Connection to Prometheus."""
+    try:
+        response = requests.get(prometheus_url, timeout=5)
+        if response.status_code == SUCCESS_STATUS:
+            return
+        log.error(
+            "Failed to connect to Prometheus Pushgateway: %s %s",
+            response.status_code,
+            response.reason,
+        )
+    except requests.exceptions.RequestException as e:
+        log.error("Error connecting to Prometheus Pushgateway: %s", str(e))
+    raise ConnectionError(
+        f"Failed to connect to Prometheus Pushgateway at {prometheus_url}",
+    )

tests/unit/job_test.py

@@ -44,9 +44,9 @@ async def test_metrics_collection(self, mock_metrics_push, *_):
             await test_job.run()
             mock_metrics_push.assert_not_called()
 
-        with patch("src.metrics.env", return_value="http://localhost:9090"):
+        with patch("src.metrics.env", return_value="http://localhost:9091"):
             await test_job.run()
             mock_metrics_push.assert_called_once()
             call_kwargs = mock_metrics_push.mock_calls[0].kwargs
-            self.assertEqual("http://localhost:9090", call_kwargs["gateway"])
+            self.assertEqual("http://localhost:9091", call_kwargs["gateway"])
             self.assertEqual("dune-sync-job name", call_kwargs["job"])