Set a timeout for the ExporterScrape view

Sometimes, users click the Test button on an Exporter and don't see a response on the UI for a
while. This happens because there are too many Hosts registered to the Project, and each Exporter
is returning a large amount of data. If Promgen is running inside a Gunicorn server, this issue
could cause the server to exit.

Therefore, we set a timeout for the ThreadPoolExecutor that the ExporterScrape view is using to
perform API calls. This will be the maximum time to wait for all scraping operations to complete.
After reaching the timeout limit, the UI will still display the results of the calls made and also
add an error message to inform the user that something went wrong.

Author: hoangpn

promgen/settings.py

@@ -224,6 +224,9 @@
     "guardian.backends.ObjectPermissionBackend",
 )
 
+# Maximum time to wait for all scraping operations to complete (in seconds)
+PROMGEN_EXPORTER_SCRAPE_TIMEOUT = env.int("PROMGEN_EXPORTER_SCRAPE_TIMEOUT", default=25)
+
 # Load overrides from PROMGEN to replace Django settings
 for k, v in PROMGEN.pop("django", {}).items():
     globals()[k] = v

promgen/static/js/promgen.vue.js

@@ -388,7 +388,17 @@ app.component('exporter-test', {
             let form = new FormData(event.srcElement.closest('form'))
             fetch(this.href, { body: form, method: "post", })
                 .then(result => result.json())
-                .then(result => exporterTestResultStore.setResults(result))
+                .then((result) => {
+                    exporterTestResultStore.setResults(result);
+                    if (result.error) {
+                        globalStore.setMessages([
+                            {
+                                class: "alert alert-danger",
+                                message: result.error,
+                            },
+                        ]);
+                    }
+                })
                 .catch(error => alert(error))
         }
     }

promgen/views.py

@@ -927,29 +927,36 @@ def query():
                             "{scheme}://{host}:{port}{path}".format(host=host.name, **data),
                         )
                     )
-                for future in concurrent.futures.as_completed(futures):
-                    try:
-                        result = future.result()
-                        result.raise_for_status()
-                        metrics = list(text_string_to_metric_families(result.text))
-                        yield (
-                            result.url,
-                            {
-                                "status_code": result.status_code,
-                                "metric_count": len(list(metrics)),
-                            },
-                        )
-                    except ValueError as e:
-                        yield result.url, f"Unable to parse metrics: {e}"
-                    except requests.ConnectionError as e:
-                        logger.warning("Error connecting to server")
-                        yield e.request.url, "Error connecting to server"
-                    except requests.RequestException as e:
-                        logger.warning("Error with response")
-                        yield e.request.url, str(e)
-                    except Exception:
-                        logger.exception("Unknown Exception")
-                        yield "Unknown URL", "Unknown error"
+                try:
+                    for future in concurrent.futures.as_completed(
+                        futures, timeout=settings.PROMGEN_EXPORTER_SCRAPE_TIMEOUT
+                    ):
+                        try:
+                            result = future.result()
+                            result.raise_for_status()
+                            metrics = list(text_string_to_metric_families(result.text))
+                            yield (
+                                result.url,
+                                {
+                                    "status_code": result.status_code,
+                                    "metric_count": len(list(metrics)),
+                                },
+                            )
+                        except ValueError as e:
+                            yield result.url, f"Unable to parse metrics: {e}"
+                        except requests.ConnectionError as e:
+                            logger.warning("Error connecting to server")
+                            yield e.request.url, "Error connecting to server"
+                        except requests.RequestException as e:
+                            logger.warning("Error with response")
+                            yield e.request.url, str(e)
+                        except Exception:
+                            logger.exception("Unknown Exception")
+                            yield "Unknown URL", "Unknown error"
+                except concurrent.futures.TimeoutError:
+                    for future in futures:
+                        future.cancel()
+                    yield "error", "Scrape timed out. Some requests may not have completed."
 
         try:
             return JsonResponse(dict(query()))