From 25da625c9e57c03f19989577f0a6340fee5d14d7 Mon Sep 17 00:00:00 2001 From: Hayk Davtyan <46712946+hayk96@users.noreply.github.com> Date: Sun, 30 Jun 2024 21:59:24 +0400 Subject: [PATCH] Release v0.4.1 (#46) * feature(ui): Add Reporting page web UI (#43) #patch Signed-off-by: hayk96 --------- Signed-off-by: hayk96 --- CHANGELOG.md | 9 + src/api/v1/endpoints/export.py | 20 ++- src/api/v1/endpoints/web.py | 25 ++- src/core/export.py | 82 +++++++-- src/models/export.py | 29 +++- src/schemas/export.json | 7 + src/utils/openapi.py | 2 +- ui/homepage/index.html | 5 + ui/metrics-management/index.html | 4 + ui/reports/index.html | 92 ++++++++++ ui/reports/script.js | 90 ++++++++++ ui/reports/style.css | 286 +++++++++++++++++++++++++++++++ ui/rules-management/index.html | 4 + 13 files changed, 631 insertions(+), 24 deletions(-) create mode 100644 ui/reports/index.html create mode 100644 ui/reports/script.js create mode 100644 ui/reports/style.css diff --git a/CHANGELOG.md b/CHANGELOG.md index 6858b39..4e12c66 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +## 0.4.1 / 2024-06-30 + +* [ENHANCEMENT] Added a new web page for reports. This page allows exporting Prometheus data in various formats directly from the web UI. #43 +* [ENHANCEMENT] Added functionality to change the timestamp format while exporting data via the /export API. Previously, the default value was Unix timestamp. Now, you can choose from the following options: iso8601, rfc2822, rfc3339, friendly, and unix (default). #41 +* [ENHANCEMENT] Added a new feature that allows replacing Prometheus labels (fields) in the final dataset: CSV, JSON, etc. #39 +* [ENHANCEMENT] Added support for exporting files in multiple formats via the /export API. Supported formats include: CSV, YAML (or YML), JSON, and JSON Lines (or NDJSON). E.g., ?format=csv|yaml|yml|json|ndjson|jsonlines. #37 +* [ENHANCEMENT] Improved the functionality that generates CSV files to ensure they have unique names instead of static names, resolving issues with responses getting mixed up between users. #35 +* [BUGFIX] Fixed exception handling for replace_fields in the /export API. #43 + ## 0.4.0 / 2024-06-23 * [ENHANCEMENT] Added a new API endpoint: `/export` for exporting data from Prometheus as a CSV file. This feature allows users to export data from Prometheus easily. diff --git a/src/api/v1/endpoints/export.py b/src/api/v1/endpoints/export.py index 9ab3ed9..a064526 100644 --- a/src/api/v1/endpoints/export.py +++ b/src/api/v1/endpoints/export.py @@ -64,12 +64,15 @@ async def export( Body( openapi_examples=ExportData._request_body_examples, ) - ] + ], + format: str = "csv" ): data = data.dict() - filename = "data.csv" expr, start = data.get("expr"), data.get("start") end, step = data.get("end"), data.get("step") + file, file_format = None, format.lower() + custom_fields, timestamp_format = data.get( + "replace_fields"), data.get("timestamp_format") validation_status, response.status_code, sts, msg = exp.validate_request( "export.json", data) if validation_status: @@ -79,9 +82,10 @@ async def export( query=expr, start=start, end=end, step=step) if resp_status: - labels, data_processed = exp.data_processor(source_data=resp_data) - csv_generator_status, sts, msg = exp.csv_generator( - data=data_processed, fields=labels, filename=filename) + labels, data_processed = exp.data_processor( + source_data=resp_data, custom_fields=custom_fields, timestamp_format=timestamp_format) + file_generator_status, sts, response.status_code, file, msg = exp.file_generator( + file_format=file_format, data=data_processed, fields=labels) else: sts, msg = resp_data.get("status"), resp_data.get("error") @@ -91,8 +95,8 @@ async def export( "status": response.status_code, "query": expr, "method": request.method, - "request_path": request.url.path}) + "request_path": f"{request.url.path}{'?' + request.url.query if request.url.query else ''}"}) if sts == "success": - return FileResponse(path=filename, - background=BackgroundTask(exp.cleanup_files, filename)) + return FileResponse(path=file, + background=BackgroundTask(exp.cleanup_files, file)) return {"status": sts, "query": expr, "message": msg} diff --git a/src/api/v1/endpoints/web.py b/src/api/v1/endpoints/web.py index 1b2d844..2a5c6cd 100644 --- a/src/api/v1/endpoints/web.py +++ b/src/api/v1/endpoints/web.py @@ -10,6 +10,7 @@ if arg_parser().get("web.enable_ui") == "true": rules_management = "ui/rules-management" metrics_management = "ui/metrics-management" + reports = "ui/reports" logger.info("Starting web management UI") @router.get("/", response_class=HTMLResponse, @@ -41,7 +42,7 @@ async def rules_management_files(path, request: Request): return f"{sts} {msg}" @router.get("/metrics-management", - description="RRenders metrics management HTML page of this application", + description="Renders metrics management HTML page of this application", include_in_schema=False) async def metrics_management_page(): return FileResponse(f"{metrics_management}/index.html") @@ -61,3 +62,25 @@ async def metrics_management_files(path, request: Request): "method": request.method, "request_path": request.url.path}) return f"{sts} {msg}" + + @router.get("/reports", + description="Renders Reports HTML page of this application", + include_in_schema=False) + async def reports_page(): + return FileResponse(f"{reports}/index.html") + + @router.get( + "/reports/{path}", + description="Returns JavaScript and CSS files of the Reports", + include_in_schema=False) + async def reports_files(path, request: Request): + if path in ["script.js", "style.css"]: + return FileResponse(f"{reports}/{path}") + sts, msg = "404", "Not Found" + logger.info( + msg=msg, + extra={ + "status": sts, + "method": request.method, + "request_path": request.url.path}) + return f"{sts} {msg}" diff --git a/src/core/export.py b/src/core/export.py index 8116f2e..3319bee 100644 --- a/src/core/export.py +++ b/src/core/export.py @@ -1,7 +1,11 @@ from jsonschema import validate, exceptions from src.utils.arguments import arg_parser +from email.utils import formatdate +from datetime import datetime +from uuid import uuid4 import requests import json +import yaml import copy import csv import os @@ -31,7 +35,41 @@ def prom_query(query, range_query=False, start="0", end="0", return True if r.status_code == 200 else False, r.status_code, r.json() -def data_processor(source_data: dict) -> tuple[list, list]: +def replace_fields(data, custom_fields) -> None: + """ + This function replaces (renames) the + final Prometheus labels (fields) based + on the 'replace_fields' object. + """ + for source_field, target_field in custom_fields.items(): + try: + if isinstance(data, list): + data[data.index(source_field)] = target_field + elif isinstance(data, dict): + data[target_field] = data.pop(source_field) + except (ValueError, KeyError): + pass + + +def format_timestamp(timestamp, fmt) -> str: + """ + This function converts Unix timestamps + to several common time formats. + """ + timestamp_formats = { + "unix": timestamp, + "rfc2822": formatdate(timestamp, localtime=True), + "iso8601": datetime.fromtimestamp(timestamp).isoformat(), + "rfc3339": datetime.fromtimestamp(timestamp).astimezone().isoformat(timespec='milliseconds'), + "friendly": datetime.fromtimestamp(timestamp).strftime('%A, %B %d, %Y %I:%M:%S %p') + } + + return timestamp_formats[fmt] + + +def data_processor(source_data: dict, + custom_fields: dict, + timestamp_format: str) -> tuple[list, list]: """ This function preprocesses the results of the Prometheus query for future formatting. @@ -47,8 +85,10 @@ def vector_processor(): ts_labels = set(ts["metric"].keys()) unique_labels.update(ts_labels) series = ts["metric"] - series["timestamp"] = ts["value"][0] + series["timestamp"] = format_timestamp( + ts["value"][0], timestamp_format) series["value"] = ts["value"][1] + replace_fields(series, custom_fields) data_processed.append(series) def matrix_processor(): @@ -58,8 +98,10 @@ def matrix_processor(): series = ts["metric"] for idx in range(len(ts["values"])): series_nested = copy.deepcopy(series) - series_nested["timestamp"] = ts["values"][idx][0] + series_nested["timestamp"] = format_timestamp( + ts["values"][idx][0], timestamp_format) series_nested["value"] = ts["values"][idx][1] + replace_fields(series_nested, custom_fields) data_processed.append(series_nested) del series_nested @@ -70,6 +112,7 @@ def matrix_processor(): unique_labels = sorted(unique_labels) unique_labels.extend(["timestamp", "value"]) + replace_fields(unique_labels, custom_fields) return unique_labels, data_processed @@ -102,18 +145,31 @@ def cleanup_files(file) -> tuple[True, str]: return True, "File has been removed successfully" -def csv_generator(data, fields, filename) -> tuple[bool, str, str]: +def file_generator(file_format, data, fields): """ - This function generates a CSV file - based on the provided objects. + This function generates a file depending + on the provided file format/extension """ + + file_path = f"/tmp/{str(uuid4())}.{file_format}" try: - with open(filename, 'w') as csvfile: - writer = csv.DictWriter( - csvfile, fieldnames=fields, extrasaction='ignore') - writer.writeheader() - writer.writerows(data) + with open(file_path, 'w') as f: + if file_format == "csv": + writer = csv.DictWriter( + f, fieldnames=fields, extrasaction='ignore') + writer.writeheader() + writer.writerows(data) + elif file_format in ["yml", "yaml"]: + f.write(yaml.dump(data)) + elif file_format == "json": + f.write(json.dumps(data)) + elif file_format in ["ndjson", "jsonlines"]: + for i in data: + f.write(f"{json.dumps(i)}\n") + else: + cleanup_files(file_path) + return False, "error", 400, "", f"Unsupported file format '{file_format}'" except BaseException as e: - return False, "error", str(e) + return False, "error", 500, "", str(e) else: - return True, "success", "CSV file has been generated successfully" + return True, "success", 200, file_path, f"{file_format.upper()} file has been generated successfully" diff --git a/src/models/export.py b/src/models/export.py index 1747d9b..751fc15 100644 --- a/src/models/export.py +++ b/src/models/export.py @@ -7,8 +7,10 @@ class ExportData(BaseModel, extra=Extra.allow): start: Optional[str] = None end: Optional[str] = None step: Optional[str] = None + timestamp_format: Optional[str] = "unix" + replace_fields: Optional[dict] = dict() _request_body_examples = { - "Count of successful logins by users per hour in a day": { + "User logins per hour in a day": { "description": "Count of successful logins by users per hour in a day", "value": { "expr": "users_login_count{status='success'}", @@ -16,5 +18,30 @@ class ExportData(BaseModel, extra=Extra.allow): "end": "2024-01-31T23:59:59Z", "step": "1h" } + }, + "User logins per hour in a day with a user-friendly time format": { + "description": "Count of successful user logins per hour in a day with a user-friendly time format", + "value": { + "expr": "users_login_count{status='success'}", + "start": "2024-01-30T00:00:00Z", + "end": "2024-01-31T23:59:59Z", + "step": "1h", + "timestamp_format": "friendly" + } + }, + "User logins per hour with friendly time format and custom fields": { + "description": "Count of successful user logins per hour in a day " + "with a user-friendly time format and custom fields", + "value": { + "expr": "users_login_count{status='success'}", + "start": "2024-01-30T00:00:00Z", + "end": "2024-01-31T23:59:59Z", + "step": "1h", + "timestamp_format": "friendly", + "replace_fields": { + "__name__": "Name", + "timestamp": "Time" + } + } } } diff --git a/src/schemas/export.json b/src/schemas/export.json index 5a264ba..0bc71cc 100644 --- a/src/schemas/export.json +++ b/src/schemas/export.json @@ -16,6 +16,13 @@ "step": { "type": ["string", "null"], "pattern": "^((([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?|0)$" + }, + "timestamp_format": { + "type": ["string"], + "pattern": "^(unix|iso8601|rfc2822|rfc3339|friendly)$" + }, + "replace_fields": { + "type": "object" } }, "required": ["expr"], diff --git a/src/utils/openapi.py b/src/utils/openapi.py index d8a3ebf..51cb221 100644 --- a/src/utils/openapi.py +++ b/src/utils/openapi.py @@ -16,7 +16,7 @@ def openapi(app: FastAPI): "providing additional features and addressing its limitations. " "Running as a sidecar alongside the Prometheus server enables " "users to extend the capabilities of the API.", - version="0.4.0", + version="0.4.1", contact={ "name": "Hayk Davtyan", "url": "https://hayk96.github.io", diff --git a/ui/homepage/index.html b/ui/homepage/index.html index ddd10b5..c9095eb 100644 --- a/ui/homepage/index.html +++ b/ui/homepage/index.html @@ -117,6 +117,7 @@ color: #ffffff; background-image: linear-gradient(45deg, #f6d365, #fda085); box-shadow: 0 3px 6px rgba(0, 0, 0, 0.16); + width: 200px; } button:hover { animation: buttonPulse 0.5s ease; @@ -170,6 +171,7 @@

The easiest Prometheus management interface

+ diff --git a/ui/metrics-management/index.html b/ui/metrics-management/index.html index 4c38e60..3fcbb1f 100644 --- a/ui/metrics-management/index.html +++ b/ui/metrics-management/index.html @@ -21,6 +21,10 @@ Rules Management Rules Management + + Reports + Reports +
diff --git a/ui/reports/index.html b/ui/reports/index.html new file mode 100644 index 0000000..c795b45 --- /dev/null +++ b/ui/reports/index.html @@ -0,0 +1,92 @@ + + + + + + Data Reports + + + + + + +
+
Export data from Prometheus as CSV, YAML, and more.
+ +
+ + + + + diff --git a/ui/reports/script.js b/ui/reports/script.js new file mode 100644 index 0000000..bad35c5 --- /dev/null +++ b/ui/reports/script.js @@ -0,0 +1,90 @@ +document.addEventListener('DOMContentLoaded', function() { + const modal = document.getElementById("exportModal"); + const btn = document.getElementById("openModalBtn"); + const span = document.getElementsByClassName("close")[0]; + + + modal.style.display = "none"; + + btn.onclick = function() { + modal.style.display = "flex"; + } + + span.onclick = function() { + modal.style.display = "none"; + } + + document.getElementById('exportForm').addEventListener('submit', async function(event) { + event.preventDefault(); + + const expr = document.getElementById('expr').value; + const start = document.getElementById('start').value ? new Date(document.getElementById('start').value).toISOString() : null; + const end = document.getElementById('end').value ? new Date(document.getElementById('end').value).toISOString() : null; + const step = document.getElementById('step').value; + const timestamp_format = document.getElementById('timestamp_format').value; + const format = document.getElementById('format').value; + + const replaceFields = {}; + document.querySelectorAll('.replace-field').forEach(field => { + const key = field.querySelector('.replace-key').value; + const value = field.querySelector('.replace-value').value; + if (key && value) { + replaceFields[key] = value; + } + }); + + const data = { + expr: expr, + start: start, + end: end, + step: step, + timestamp_format: timestamp_format, + replace_fields: replaceFields + }; + + try { + const response = await fetch('/api/v1/export?format=' + format, { + method: 'POST', + headers: { + 'Content-Type': 'application/json' + }, + body: JSON.stringify(data) + }); + + if (!response.ok) { + throw new Error(`Error: ${response.statusText}`); + } + + const blob = await response.blob(); + const url = window.URL.createObjectURL(blob); + const a = document.createElement('a'); + a.style.display = 'none'; + a.href = url; + a.download = `data.${format}`; + document.body.appendChild(a); + a.click(); + window.URL.revokeObjectURL(url); + document.body.removeChild(a); + } catch (error) { + document.getElementById('results').innerHTML = `

${error.message}

`; + } + }); + + document.getElementById('addReplaceField').addEventListener('click', function() { + const container = document.getElementById('replaceFieldsContainer'); + const newField = document.createElement('div'); + newField.className = 'replace-field'; + newField.innerHTML = ` + + + + `; + container.appendChild(newField); + }); + + document.getElementById('replaceFieldsContainer').addEventListener('click', function(event) { + if (event.target.classList.contains('remove-field')) { + event.target.parentElement.remove(); + } + }); +}); diff --git a/ui/reports/style.css b/ui/reports/style.css new file mode 100644 index 0000000..9b8f485 --- /dev/null +++ b/ui/reports/style.css @@ -0,0 +1,286 @@ +body { + font-family: Arial, sans-serif; + background-color: #f5f5f5; + color: #333; + margin: 0; + padding: 0; + display: flex; + flex-direction: column; +} + + +.sidebar { + position: fixed; + left: 0; + top: 0; + height: 100%; + width: 60px; + background-color: #30354b; + transition: width 0.3s ease; + z-index: 1001; + overflow: hidden; + display: flex; + flex-direction: column; +} + +.sidebar:hover { + width: 220px; +} + + +.sidebar-icon { + display: flex; + align-items: center; + padding: 20px; + color: white; + text-decoration: none; + overflow: hidden; + white-space: nowrap; +} + +.sidebar-icon img { + width: 30px; + transition: transform 0.2s ease; +} + +.icon-label { + margin-left: 10px; + opacity: 0; + transition: opacity 0.3s ease, margin-left 0.3s ease; + white-space: nowrap; +} + +.sidebar:hover .icon-label { + opacity: 1; + margin-left: 10px; +} + + +.main-content { + margin-left: 60px; + padding: 20px; + width: 100%; + transition: margin-left 0.3s ease; +} + +.sidebar:hover + .main-content { + margin-left: 220px; +} + +h1 { + font-size: 24px; + margin-bottom: 20px; +} + +button { + padding: 10px 20px; + background-color: #48BB78; + color: white; + border: none; + border-radius: 4px; + cursor: pointer; + font-size: 16px; + font-weight: bold; + transition: background-color 0.3s ease; + margin-bottom: 10px; +} + +button:hover { + background-color: #38A169; +} + +#results { + margin-top: 20px; +} + + +.modal { + display: none; + position: fixed; + z-index: 1000; + left: 0; + top: 0; + width: 100%; + height: 100%; + overflow: hidden; + background-color: rgba(0,0,0,0.4); + display: flex; + align-items: center; + justify-content: center; +} + +.modal-content { + background-color: #fff; + padding: 20px; + border: 1px solid #888; + width: 80%; + max-width: 600px; + border-radius: 10px; + box-shadow: 0 4px 8px rgba(0,0,0,0.1); + overflow: auto; + max-height: 90%; +} + +.close { + color: #aaa; + float: right; + font-size: 28px; + font-weight: bold; +} + +.close:hover, +.close:focus { + color: black; + text-decoration: none; + cursor: pointer; +} + +.form-group { + margin-bottom: 15px; +} + +label { + display: block; + margin-bottom: 5px; + font-weight: bold; +} + +input, select, textarea { + width: 100%; + padding: 8px; + border: 1px solid #ccc; + border-radius: 4px; + box-sizing: border-box; +} + +textarea { + resize: vertical; + min-height: 100px; +} + +.optional { + font-weight: normal; + color: #666; + font-style: italic; +} + +.replace-field { + display: flex; + align-items: center; + gap: 10px; + margin-bottom: 10px; +} + +.replace-field input { + flex: 1; +} + +.remove-field { + padding: 5px; + background-color: #f44336; + color: white; + border: none; + border-radius: 50%; + cursor: pointer; + font-size: 16px; + font-weight: bold; + width: 22px; + height: 22px; + line-height: 10%; + text-align: center; + transition: background-color 0.3s ease; +} + +.remove-field:hover { + background-color: #d32f2f; +} + +#addReplaceField { + padding: 5px; + background-color: #48BB78; + color: white; + border: none; + border-radius: 50%; + cursor: pointer; + font-size: 16px; + font-weight: bold; + width: 22px; + height: 22px; + line-height: 50%; + text-align: center; + transition: background-color 0.3s ease; + margin-top: 10px; +} + +#addReplaceField:hover { + background-color: #38A169; +} + +.submit-btn { + display: inline-block; + padding: 10px 20px; + background-color: #48BB78; + color: white; + border: none; + border-radius: 4px; + cursor: pointer; + font-size: 16px; + font-weight: bold; + transition: background-color 0.3s ease; +} + +.submit-btn:hover { + background-color: #38A169; +} + + +@media (max-width: 768px) { + .sidebar { + width: 100%; + height: 60px; + bottom: 0; + top: auto; + display: flex; + justify-content: space-between; + align-items: center; + flex-direction: row; + } + + .sidebar:hover { + width: 100%; + height: 60px; + flex-direction: row; + } + + .sidebar-icon { + padding: 10px; + flex: 1; + justify-content: center; + } + + .icon-label { + display: none; + } + + .main-content { + margin-left: 0; + margin-top: 60px; + width: 100%; + } + + .sidebar:hover + .main-content { + margin-left: 0; + } +} + +@media (max-width: 480px) { + .modal-content { + width: 90%; + padding: 10px; + } + + button { + padding: 8px 16px; + font-size: 14px; + } +} diff --git a/ui/rules-management/index.html b/ui/rules-management/index.html index 3336599..3a2ed9a 100644 --- a/ui/rules-management/index.html +++ b/ui/rules-management/index.html @@ -23,6 +23,10 @@ Metrics Management Metrics Management + + Reports + Reports +