Skip to content

Commit 3a6a189

Browse files
committed
Set a timeout for the ExporterScrape view
Sometimes, users click the Test button on an Exporter and don't see a response on the UI for a while. This happens because there are too many Hosts registered to the Project, and each Exporter is returning a large amount of data. If Promgen is running inside a Gunicorn server, this issue could cause the server to exit. Therefore, we set a timeout for the ThreadPoolExecutor that the ExporterScrape view is using to perform API calls. This will be the maximum time to wait for all scraping operations to complete. After reaching the timeout limit, the UI will still display the results of the calls made and also add an error message to inform the user that something went wrong.
1 parent 914c23c commit 3a6a189

3 files changed

Lines changed: 44 additions & 24 deletions

File tree

promgen/settings.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -224,6 +224,9 @@
224224
"guardian.backends.ObjectPermissionBackend",
225225
)
226226

227+
# Maximum time to wait for all scraping operations to complete (in seconds)
228+
PROMGEN_EXPORTER_SCRAPE_TIMEOUT = env.int("PROMGEN_EXPORTER_SCRAPE_TIMEOUT", default=25)
229+
227230
# Load overrides from PROMGEN to replace Django settings
228231
for k, v in PROMGEN.pop("django", {}).items():
229232
globals()[k] = v

promgen/static/js/promgen.vue.js

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -388,7 +388,17 @@ app.component('exporter-test', {
388388
let form = new FormData(event.srcElement.closest('form'))
389389
fetch(this.href, { body: form, method: "post", })
390390
.then(result => result.json())
391-
.then(result => exporterTestResultStore.setResults(result))
391+
.then((result) => {
392+
exporterTestResultStore.setResults(result);
393+
if (result.error) {
394+
globalStore.setMessages([
395+
{
396+
class: "alert alert-danger",
397+
message: result.error,
398+
},
399+
]);
400+
}
401+
})
392402
.catch(error => alert(error))
393403
}
394404
}

promgen/views.py

Lines changed: 30 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -927,29 +927,36 @@ def query():
927927
"{scheme}://{host}:{port}{path}".format(host=host.name, **data),
928928
)
929929
)
930-
for future in concurrent.futures.as_completed(futures):
931-
try:
932-
result = future.result()
933-
result.raise_for_status()
934-
metrics = list(text_string_to_metric_families(result.text))
935-
yield (
936-
result.url,
937-
{
938-
"status_code": result.status_code,
939-
"metric_count": len(list(metrics)),
940-
},
941-
)
942-
except ValueError as e:
943-
yield result.url, f"Unable to parse metrics: {e}"
944-
except requests.ConnectionError as e:
945-
logger.warning("Error connecting to server")
946-
yield e.request.url, "Error connecting to server"
947-
except requests.RequestException as e:
948-
logger.warning("Error with response")
949-
yield e.request.url, str(e)
950-
except Exception:
951-
logger.exception("Unknown Exception")
952-
yield "Unknown URL", "Unknown error"
930+
try:
931+
for future in concurrent.futures.as_completed(
932+
futures, timeout=settings.PROMGEN_EXPORTER_SCRAPE_TIMEOUT
933+
):
934+
try:
935+
result = future.result()
936+
result.raise_for_status()
937+
metrics = list(text_string_to_metric_families(result.text))
938+
yield (
939+
result.url,
940+
{
941+
"status_code": result.status_code,
942+
"metric_count": len(list(metrics)),
943+
},
944+
)
945+
except ValueError as e:
946+
yield result.url, f"Unable to parse metrics: {e}"
947+
except requests.ConnectionError as e:
948+
logger.warning("Error connecting to server")
949+
yield e.request.url, "Error connecting to server"
950+
except requests.RequestException as e:
951+
logger.warning("Error with response")
952+
yield e.request.url, str(e)
953+
except Exception:
954+
logger.exception("Unknown Exception")
955+
yield "Unknown URL", "Unknown error"
956+
except concurrent.futures.TimeoutError:
957+
for future in futures:
958+
future.cancel()
959+
yield "error", "Scrape timed out. Some requests may not have completed."
953960

954961
try:
955962
return JsonResponse(dict(query()))

0 commit comments

Comments
 (0)