From 647991d8a6c4d696702e96f7aaf5ddc0b71cabb4 Mon Sep 17 00:00:00 2001 From: Mamoutou Diarra Date: Mon, 13 Jul 2026 21:25:53 +0200 Subject: [PATCH 1/2] add dst dashboard frontend --- .vscode/settings.json | 4 - dst_dashboard/.dockerignore | 59 -- dst_dashboard/Dockerfile | 28 - dst_dashboard/__init__.py | 1 - dst_dashboard/api/__init__.py | 1 - dst_dashboard/api/admin.py | 157 ---- dst_dashboard/api/datasets.py | 151 ---- dst_dashboard/api/datasources.py | 58 -- dst_dashboard/api/experiments.py | 209 ----- dst_dashboard/api/panels.py | 188 ----- dst_dashboard/api/utils.py | 29 - dst_dashboard/auth.py | 56 -- dst_dashboard/config/__init__.py | 1 - dst_dashboard/config/constants.py | 36 - dst_dashboard/config/data_structures.py | 125 --- dst_dashboard/config/utils.py | 13 - dst_dashboard/k8s/backend.yaml | 155 ---- dst_dashboard/k8s/frontend.yaml | 84 -- dst_dashboard/k8s/mongodb-dev.yaml | 79 -- dst_dashboard/k8s/mongodb-prod.yaml | 79 -- dst_dashboard/main.py | 86 --- dst_dashboard/processors/__init__.py | 1 - dst_dashboard/processors/dataset_processor.py | 374 --------- .../processors/experiment_processor.py | 131 ---- dst_dashboard/processors/panel_processor.py | 500 ------------ dst_dashboard/processors/tests/__init__.py | 0 .../processors/tests/test_panel_processor.py | 723 ------------------ dst_dashboard/sample_config.yaml | 23 - dst_dashboard/scripts/__init__.py | 0 .../scripts/seed_data/experiments.json | 367 --------- dst_dashboard/scripts/seed_experiments.py | 94 --- dst_dashboard/static/admin.html | 384 ---------- dst_dashboard/storage/__init__.py | 1 - dst_dashboard/storage/db.py | 232 ------ 34 files changed, 4429 deletions(-) delete mode 100644 .vscode/settings.json delete mode 100644 dst_dashboard/.dockerignore delete mode 100644 dst_dashboard/Dockerfile delete mode 100644 dst_dashboard/__init__.py delete mode 100644 dst_dashboard/api/__init__.py delete mode 100644 dst_dashboard/api/admin.py delete mode 100644 dst_dashboard/api/datasets.py delete mode 100644 dst_dashboard/api/datasources.py delete mode 100644 dst_dashboard/api/experiments.py delete mode 100644 dst_dashboard/api/panels.py delete mode 100644 dst_dashboard/api/utils.py delete mode 100644 dst_dashboard/auth.py delete mode 100644 dst_dashboard/config/__init__.py delete mode 100644 dst_dashboard/config/constants.py delete mode 100644 dst_dashboard/config/data_structures.py delete mode 100644 dst_dashboard/config/utils.py delete mode 100644 dst_dashboard/k8s/backend.yaml delete mode 100644 dst_dashboard/k8s/frontend.yaml delete mode 100644 dst_dashboard/k8s/mongodb-dev.yaml delete mode 100644 dst_dashboard/k8s/mongodb-prod.yaml delete mode 100644 dst_dashboard/main.py delete mode 100644 dst_dashboard/processors/__init__.py delete mode 100644 dst_dashboard/processors/dataset_processor.py delete mode 100644 dst_dashboard/processors/experiment_processor.py delete mode 100644 dst_dashboard/processors/panel_processor.py delete mode 100644 dst_dashboard/processors/tests/__init__.py delete mode 100644 dst_dashboard/processors/tests/test_panel_processor.py delete mode 100644 dst_dashboard/sample_config.yaml delete mode 100644 dst_dashboard/scripts/__init__.py delete mode 100644 dst_dashboard/scripts/seed_data/experiments.json delete mode 100644 dst_dashboard/scripts/seed_experiments.py delete mode 100644 dst_dashboard/static/admin.html delete mode 100644 dst_dashboard/storage/__init__.py delete mode 100644 dst_dashboard/storage/db.py diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index 4b5a2944..00000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "python-envs.defaultEnvManager": "ms-python.python:conda", - "python-envs.defaultPackageManager": "ms-python.python:conda" -} \ No newline at end of file diff --git a/dst_dashboard/.dockerignore b/dst_dashboard/.dockerignore deleted file mode 100644 index 91dff79f..00000000 --- a/dst_dashboard/.dockerignore +++ /dev/null @@ -1,59 +0,0 @@ -# Python -__pycache__ -*.py[cod] -*$py.class -*.so -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -*.egg-info/ -.installed.cfg -*.egg -MANIFEST - -# Virtual environments -venv/ -ENV/ -env/ - -# Testing -.pytest_cache/ -.coverage -htmlcov/ - -# IDE -.vscode/ -.idea/ -*.swp -*.swo - -# OS -.DS_Store -Thumbs.db - -# Git -.git/ -.gitignore - -# Node (for frontend) -node_modules/ -dst_dashboard/frontend/node_modules/ -dst_dashboard/frontend/build/ - -# Database -*.db -*.sqlite -.cache/ - -# Logs -*.log diff --git a/dst_dashboard/Dockerfile b/dst_dashboard/Dockerfile deleted file mode 100644 index 8f8682fb..00000000 --- a/dst_dashboard/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -FROM python:3.11-slim - -WORKDIR /app - -# Install system dependencies -RUN apt-get update && apt-get install -y \ - build-essential \ - && rm -rf /var/lib/apt/lists/* - -# Copy project files -COPY pyproject.toml . -COPY src/ ./src/ -COPY dst_dashboard/ ./dst_dashboard/ - -# Install uv for faster dependency installation -RUN pip install --no-cache-dir uv - -# Install dependencies -RUN uv pip install --system --no-cache -r pyproject.toml - -# Create database directory -RUN mkdir -p /app/data - -# Expose port -EXPOSE 8000 - -# Run the application -CMD ["uvicorn", "dst_dashboard.main:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/dst_dashboard/__init__.py b/dst_dashboard/__init__.py deleted file mode 100644 index 3dc1f76b..00000000 --- a/dst_dashboard/__init__.py +++ /dev/null @@ -1 +0,0 @@ -__version__ = "0.1.0" diff --git a/dst_dashboard/api/__init__.py b/dst_dashboard/api/__init__.py deleted file mode 100644 index d93703a9..00000000 --- a/dst_dashboard/api/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""FastAPI REST for DST Dashboard.""" diff --git a/dst_dashboard/api/admin.py b/dst_dashboard/api/admin.py deleted file mode 100644 index d5fedf81..00000000 --- a/dst_dashboard/api/admin.py +++ /dev/null @@ -1,157 +0,0 @@ -"""Admin API routes for configuration and data management.""" - -import logging -from pathlib import Path - -from fastapi import APIRouter, Depends, HTTPException, Request -from fastapi.responses import HTMLResponse - -from dst_dashboard.auth import create_admin_token, require_admin_token -from dst_dashboard.config.utils import LoadConfig -from dst_dashboard.processors.experiment_processor import ExperimentProcessor -from dst_dashboard.storage.db import DSTDatabase - -router = APIRouter(prefix="/admin", tags=["admin"]) -logger = logging.getLogger(__name__) - -_ADMIN_HTML_PATH = Path(__file__).parent.parent / "static" / "admin.html" - - -@router.post("/reload") -def reload_config(request: Request, _: None = Depends(require_admin_token)): - """Reload datasources from config.yaml (experiments are API-managed and unaffected).""" - try: - logger.info("Reloading configuration...") - - # Load and validate configuration - config = LoadConfig().WithValidateDatasources() - logger.info(f"Reloaded configuration with {len(config.datasources)} datasources") - - # Update app state - request.app.state.config = config - request.app.state.datasources = config.datasources - - # Initialize database - db = DSTDatabase() - - # Clear and re-insert datasources - db.datasources.delete_many({}) - db.insert_datasource_list(config.datasources) - logger.info(f"Re-inserted {len(config.datasources)} datasources") - - return { - "status": "success", - "message": "Datasources reloaded successfully", - "summary": { - "datasources": len(config.datasources), - }, - } - - except Exception as e: - logger.error(f"Failed to reload configuration: {e}", exc_info=True) - raise HTTPException(status_code=500, detail=f"Failed to reload configuration: {str(e)}") - - -@router.delete("/datasets/{experiment_id}/{dataset_name}") -def clear_dataset(experiment_id: str, dataset_name: str, _: None = Depends(require_admin_token)): - """Clear cached dataset data to force re-fetch on next request.""" - try: - db = DSTDatabase() - - if db.delete_dataset(experiment_id, dataset_name): - logger.info(f"Cleared dataset: {experiment_id}:{dataset_name}") - return { - "status": "success", - "message": f"Dataset '{dataset_name}' cleared successfully", - } - else: - raise HTTPException(status_code=404, detail=f"Dataset '{dataset_name}' not found") - - except HTTPException: - raise - except Exception as e: - logger.error(f"Failed to clear dataset: {e}") - raise HTTPException(status_code=500, detail=f"Failed to clear dataset: {str(e)}") - - -@router.delete("/datasets") -def clear_all_datasets(_: None = Depends(require_admin_token)): - """Clear all cached dataset data to force re-fetch.""" - try: - db = DSTDatabase() - count = db.clear_all_dataset_cache() - - logger.info(f"Cleared {count} datasets") - - return {"status": "success", "message": f"Cleared {count} datasets successfully"} - - except Exception as e: - logger.error(f"Failed to clear datasets: {e}") - raise HTTPException(status_code=500, detail=f"Failed to clear datasets: {str(e)}") - - -@router.post("/experiments/{experiment_id}/reprocess") -def reprocess_experiment( - experiment_id: str, request: Request, _: None = Depends(require_admin_token) -): - """Reprocess a single experiment - fetch datasets and regenerate panels.""" - try: - db = DSTDatabase() - - # Get experiment from database - experiment_data = db.get_experiment(experiment_id) - if not experiment_data: - raise HTTPException(status_code=404, detail="Experiment not found") - - # Get config from app state - config = request.app.state.config - - # Initialize processor - from dst_dashboard.config.data_structures import ExperimentConfig - - processor = ExperimentProcessor(config, db) - - experiment = ExperimentConfig(**experiment_data) - - logger.info(f"Reprocessing experiment: {experiment_id}") - - # Process the experiment (this will fetch datasets and transform panels) - success = processor.process_experiment(experiment) - - if success: - return { - "status": "success", - "message": f"Experiment '{experiment_id}' reprocessed successfully", - "experiment_id": experiment_id, - "datasets_count": len(experiment.datasets), - "panels_count": len(experiment.panels), - } - else: - raise HTTPException( - status_code=500, detail=f"Failed to reprocess experiment '{experiment_id}'" - ) - - except HTTPException: - raise - except Exception as e: - logger.error(f"Failed to reprocess experiment: {e}", exc_info=True) - raise HTTPException(status_code=500, detail=f"Failed to reprocess experiment: {str(e)}") - - -@router.get("/token", response_class=HTMLResponse) -def admin_page(): - """ - Serve the admin page: a "Generate token" button, plus a small UI for - creating/editing/deleting experiments by pasting JSON. - - Not gated by require_admin_token - you need this page to get a token in - the first place. Access is instead controlled upstream by Authentik - forward-auth on this ingress path (which covers both GET and POST here). - """ - return HTMLResponse(content=_ADMIN_HTML_PATH.read_text()) - - -@router.post("/token") -def generate_admin_token(): - """Mint a fresh admin-scope token. Same protection boundary as GET /admin/token.""" - return {"token": create_admin_token()} diff --git a/dst_dashboard/api/datasets.py b/dst_dashboard/api/datasets.py deleted file mode 100644 index d9d06ab5..00000000 --- a/dst_dashboard/api/datasets.py +++ /dev/null @@ -1,151 +0,0 @@ -"""Dataset API routes.""" - -from fastapi import APIRouter, Depends, HTTPException, Request - -from dst_dashboard.api.utils import get_processor -from dst_dashboard.auth import require_admin_token -from dst_dashboard.config.data_structures import DatasetConfig, ExperimentConfig -from dst_dashboard.storage.db import DSTDatabase - -router = APIRouter(prefix="/experiments/{experiment_id}/datasets", tags=["datasets"]) - - -@router.get("") -def get_experiment_datasets(experiment_id: str, request: Request): - """Get all datasets for an experiment with their data.""" - db = DSTDatabase() - - # Get experiment from database (source of truth) - experiment_data = db.get_experiment(experiment_id) - if experiment_data is None: - raise HTTPException(status_code=404, detail="Experiment not found") - - experiment = ExperimentConfig(**experiment_data) - - # Initialize database - db = DSTDatabase() - - # Get all datasets with their data - datasets = [] - for dataset_config in experiment.datasets: - cached_data = db.get_dataset(experiment_id, dataset_config.name) - - datasets.append( - { - "name": dataset_config.name, - "datasource": dataset_config.datasource, - "timeRange": { - "start": dataset_config.timeRange.start.isoformat(), - "end": dataset_config.timeRange.end.isoformat(), - }, - "schema": [{"name": f.name, "type": f.type} for f in dataset_config.schema], - "rowCount": len(cached_data) if cached_data else 0, - "data": cached_data if cached_data else [], - } - ) - - return {"experiment_id": experiment_id, "datasets": datasets} - - -@router.get("/{dataset_name}", response_model=DatasetConfig) -def get_dataset(experiment_id: str, dataset_name: str, request: Request): - """Get dataset configuration for an experiment.""" - db = DSTDatabase() - - # Get experiment from database - experiment_data = db.get_experiment(experiment_id) - if experiment_data is None: - raise HTTPException(status_code=404, detail="Experiment not found") - - experiment = ExperimentConfig(**experiment_data) - - # Find dataset - for dataset in experiment.datasets: - if dataset.name == dataset_name: - return dataset - - raise HTTPException(status_code=404, detail="Dataset not found") - - -@router.get("/{dataset_name}/data") -def get_dataset_data( - experiment_id: str, dataset_name: str, request: Request, refresh: bool = False -): - """Get dataset data (with caching support). `refresh=True` forces a re-fetch.""" - db = DSTDatabase() - - # Get experiment from database - experiment_data = db.get_experiment(experiment_id) - if experiment_data is None: - raise HTTPException(status_code=404, detail="Experiment not found") - - experiment = ExperimentConfig(**experiment_data) - - # Find dataset config - dataset_config = None - for dataset in experiment.datasets: - if dataset.name == dataset_name: - dataset_config = dataset - break - - if dataset_config is None: - raise HTTPException(status_code=404, detail="Dataset not found") - - # Initialize processor - processor = get_processor(request) - - # Check cache unless refresh requested - if not refresh: - cached_data = db.get_dataset(experiment_id, dataset_name) - if cached_data is not None: - return {"data": cached_data, "source": "cache"} - - # Fetch fresh data using processor - try: - data = processor.fetch_dataset(experiment_id, dataset_config) - - # Update cache - if data: - db.store_dataset(experiment_id, dataset_name, data) - - return {"data": data, "source": "datasource"} - - except Exception as e: - raise HTTPException(status_code=500, detail=f"Failed to fetch dataset: {str(e)}") - - -@router.delete("/{dataset_name}", status_code=204) -def delete_dataset( - experiment_id: str, - dataset_name: str, - request: Request, - _: None = Depends(require_admin_token), -): - """Delete a dataset and its data.""" - db = DSTDatabase() - - # Get experiment from database - experiment_data = db.get_experiment(experiment_id) - if experiment_data is None: - raise HTTPException(status_code=404, detail="Experiment not found") - - experiment = ExperimentConfig(**experiment_data) - - # Find dataset - dataset_index = None - for i, ds in enumerate(experiment.datasets): - if ds.name == dataset_name: - dataset_index = i - break - - if dataset_index is None: - raise HTTPException(status_code=404, detail="Dataset not found") - - # Remove dataset from experiment - experiment.datasets.pop(dataset_index) - db.store_experiment(experiment.model_dump()) - - # Delete dataset data - db.delete_dataset(experiment_id, dataset_name) - - return None diff --git a/dst_dashboard/api/datasources.py b/dst_dashboard/api/datasources.py deleted file mode 100644 index fe2f162c..00000000 --- a/dst_dashboard/api/datasources.py +++ /dev/null @@ -1,58 +0,0 @@ -"""Datasource API routes.""" - -from typing import List - -from fastapi import APIRouter, HTTPException, Request - -from dst_dashboard.config.data_structures import DataSourceConfig - -router = APIRouter(prefix="/datasources", tags=["datasources"]) - - -@router.get("", response_model=List[DataSourceConfig]) -def list_datasources( - request: Request, -): - """List loaded datasources""" - datasources = getattr(request.app.state, "datasources", None) - if datasources is None: - raise HTTPException(status_code=500, detail="Datasources are not initialized") - return datasources - - -@router.get("/prometheus", response_model=List[DataSourceConfig]) -def list_prometheus_datasources( - request: Request, -): - """List prometheus datasources""" - datasources = getattr(request.app.state, "datasources", None) - if datasources is None: - raise HTTPException(status_code=500, detail="Datasources are not initialized") - prometheus_datasources = [ds for ds in datasources if ds.type.lower() == "prometheus"] - return prometheus_datasources - - -@router.get("/victorialogs", response_model=List[DataSourceConfig]) -def list_victorialogs_datasources( - request: Request, -): - """List victorialogs datasources""" - datasources = getattr(request.app.state, "datasources", None) - if datasources is None: - raise HTTPException(status_code=500, detail="Datasources are not initialized") - victorialogs_datasources = [ds for ds in datasources if ds.type.lower() == "victorialogs"] - return victorialogs_datasources - - -@router.get("/{datasource_name}", response_model=DataSourceConfig) -def get_datasource(datasource_name: str, request: Request): - """Get a datasource by name from config.""" - datasources = getattr(request.app.state, "datasources", None) - if datasources is None: - raise HTTPException(status_code=500, detail="Datasources are not initialized") - - for datasource in datasources: - if datasource.name == datasource_name: - return datasource - - raise HTTPException(status_code=404, detail="Datasource not found") diff --git a/dst_dashboard/api/experiments.py b/dst_dashboard/api/experiments.py deleted file mode 100644 index cc60445a..00000000 --- a/dst_dashboard/api/experiments.py +++ /dev/null @@ -1,209 +0,0 @@ -from typing import Any, Dict, List, Optional - -from bson import ObjectId -from fastapi import APIRouter, Depends, HTTPException, Query, Request - -from dst_dashboard.api.utils import get_processor -from dst_dashboard.auth import require_admin_token -from dst_dashboard.config.data_structures import ExperimentConfig -from dst_dashboard.storage.db import DSTDatabase - -router = APIRouter(prefix="/experiments", tags=["experiments"]) - - -@router.get("/families") -def list_families(request: Request) -> Dict[str, Any]: - """ - Get all experiment families with their experiments. - Organized for navigation in the dashboard UI. - - Returns: - List of families with nested experiments - """ - db = DSTDatabase() - - # Get experiments from database (source of truth) - experiments_data = db.list_experiments() - experiments = [ExperimentConfig(**exp_data) for exp_data in experiments_data] - - # Group experiments by family - families_dict = {} - for experiment in experiments: - if not experiment.publish: - continue - - family = experiment.family - if family not in families_dict: - families_dict[family] = {"name": family, "experiments": []} - - families_dict[family]["experiments"].append( - { - "id": experiment.id, - "title": experiment.title, - "panel_count": len(experiment.panels), - "dataset_count": len(experiment.datasets), - "metadata": experiment.metadata, - "description": experiment.description, - "github_repo": experiment.github_repo, - "github_pr": experiment.github_pr, - "docker_image": experiment.docker_image, - "date": experiment.date, - } - ) - - # Convert to sorted list - families = sorted(families_dict.values(), key=lambda x: x["name"]) - - return { - "families": families, - "total_families": len(families), - "total_experiments": sum(len(f["experiments"]) for f in families), - } - - -@router.get("", response_model=List[ExperimentConfig]) -def list_experiments( - request: Request, - publish: Optional[bool] = Query(None, description="Filter by publish status"), -): - """List all experiments from database. Use ?publish=true to filter only published experiments.""" - db = DSTDatabase() - - # Get experiments from database (source of truth) - experiments_data = db.list_experiments() - experiments = [ExperimentConfig(**exp_data) for exp_data in experiments_data] - - if publish is not None: - experiments = [experiment for experiment in experiments if experiment.publish == publish] - - return experiments - - -@router.get("/{experiment_id}", response_model=ExperimentConfig) -def get_experiment(experiment_id: str, request: Request): - """Get experiment details from database.""" - db = DSTDatabase() - - # Get from database (source of truth) - experiment_data = db.get_experiment(experiment_id) - - if experiment_data is None: - raise HTTPException(status_code=404, detail="Experiment not found") - - return ExperimentConfig(**experiment_data) - - -@router.post("", response_model=ExperimentConfig, status_code=201) -def create_experiment( - experiment: ExperimentConfig, request: Request, _: None = Depends(require_admin_token) -): - """ - Create a new experiment and process it (fetch datasets, generate panels). - `id` is always server-assigned - any id in the request body is ignored. - """ - db = DSTDatabase() - - # id is never client-supplied - always assign a fresh one - experiment.id = str(ObjectId()) - - # Title must be unique - if db.experiments.find_one({"title": experiment.title}): - raise HTTPException( - status_code=409, detail=f"Experiment with title '{experiment.title}' already exists" - ) - - # Store experiment in database - db.store_experiment(experiment.model_dump()) - - # Process the experiment (fetch datasets, generate panels) - processor = get_processor(request) - success = processor.process_experiment(experiment) - - if not success: - # Rollback - delete from DB if processing failed - db.delete_experiment(experiment.id) - raise HTTPException( - status_code=500, - detail="Failed to process experiment. Check that all datasets and datasources are valid.", - ) - - return experiment - - -@router.put("/{experiment_id}", response_model=ExperimentConfig) -def update_experiment( - experiment_id: str, - experiment: ExperimentConfig, - request: Request, - _: None = Depends(require_admin_token), -): - """ - Update an existing experiment. - Only reprocesses if configuration changed (datasets, panels, datasources, time ranges). - `id` always comes from the URL path - any id in the request body is ignored. - """ - db = DSTDatabase() - - # id is never client-supplied - the path is authoritative - experiment.id = experiment_id - - # Get existing experiment from DB - existing_data = db.get_experiment(experiment_id) - if not existing_data: - raise HTTPException(status_code=404, detail="Experiment not found") - - existing = ExperimentConfig(**existing_data) - - # Title must be unique among *other* experiments - title_conflict = db.experiments.find_one( - {"title": experiment.title, "id": {"$ne": experiment_id}} - ) - if title_conflict: - raise HTTPException( - status_code=409, detail=f"Experiment with title '{experiment.title}' already exists" - ) - - # Check if reprocessing is needed - needs_reprocessing = ( - existing.datasets != experiment.datasets - or existing.panels != experiment.panels - or any( - existing_ds.datasource != new_ds.datasource or existing_ds.timeRange != new_ds.timeRange - for existing_ds, new_ds in zip(existing.datasets, experiment.datasets) - ) - if len(existing.datasets) == len(experiment.datasets) - else True - ) - - # Store updated experiment in database - db.store_experiment(experiment.model_dump()) - - # Reprocess if configuration changed - if needs_reprocessing: - processor = get_processor(request) - success = processor.process_experiment(experiment) - - if not success: - # Rollback - restore old experiment - db.store_experiment(existing.model_dump()) - raise HTTPException( - status_code=500, - detail="Failed to reprocess experiment. Rolled back to previous configuration.", - ) - - return experiment - - -@router.delete("/{experiment_id}", status_code=204) -def delete_experiment(experiment_id: str, request: Request, _: None = Depends(require_admin_token)): - """Delete an experiment and all its associated data (datasets, panels).""" - db = DSTDatabase() - - # Check if experiment exists - if not db.get_experiment(experiment_id): - raise HTTPException(status_code=404, detail="Experiment not found") - - # Delete from database (cascades to datasets and panels) - db.delete_experiment(experiment_id) - - return None diff --git a/dst_dashboard/api/panels.py b/dst_dashboard/api/panels.py deleted file mode 100644 index 60aca106..00000000 --- a/dst_dashboard/api/panels.py +++ /dev/null @@ -1,188 +0,0 @@ -"""Panel API routes.""" - -import logging -from typing import Any, Dict - -from fastapi import APIRouter, Depends, HTTPException, Request - -from dst_dashboard.auth import require_admin_token -from dst_dashboard.config.data_structures import ExperimentConfig -from dst_dashboard.storage.db import DSTDatabase - -router = APIRouter(prefix="/experiments/{experiment_id}/panels", tags=["panels"]) -logger = logging.getLogger(__name__) - - -@router.get("") -def get_all_panels(experiment_id: str, request: Request) -> Dict[str, Any]: - """Get all panels for an experiment with their rendered visualizations.""" - db = DSTDatabase() - - # Get experiment from database - experiment_data = db.get_experiment(experiment_id) - if experiment_data is None: - raise HTTPException(status_code=404, detail="Experiment not found") - - experiment = ExperimentConfig(**experiment_data) - - # Get panel processor to transform data on-demand - from dst_dashboard.api.utils import get_panel_processor - - processor = get_panel_processor(request) - - # Get stored panel data from database - rendered_panels = [] - for panel_config in experiment.panels: - # Get pre-processed panel from database - panel_data = processor.db.get_panel_data(experiment_id, panel_config.name) - - if panel_data: - rendered_panels.append( - { - "panel_name": panel_config.name, - "panel_title": panel_config.title, - "panel_type": panel_config.type, - "dataset": panel_config.dataset, - "option": panel_data, - } - ) - else: - logger.warning( - f"Panel '{panel_config.name}' not found in database, may need reprocessing" - ) - rendered_panels.append( - { - "panel_name": panel_config.name, - "panel_title": panel_config.title, - "panel_type": panel_config.type, - "dataset": panel_config.dataset, - "error": "Panel not preprocessed", - } - ) - - return {"experiment_id": experiment_id, "panels": rendered_panels} - - -@router.get("/by-dataset/{dataset_name}") -def get_panels_by_dataset( - experiment_id: str, dataset_name: str, request: Request -) -> Dict[str, Any]: - """Get all preprocessed panels that use a specific dataset.""" - db = DSTDatabase() - - # Get experiment from database - experiment_data = db.get_experiment(experiment_id) - if experiment_data is None: - raise HTTPException(status_code=404, detail="Experiment not found") - - experiment = ExperimentConfig(**experiment_data) - - # Filter panels by dataset - matching_panels = [p for p in experiment.panels if p.dataset == dataset_name] - - if not matching_panels: - return {"experiment_id": experiment_id, "dataset_name": dataset_name, "panels": []} - - # Retrieve preprocessed panels from database - rendered_panels = [] - for panel_config in matching_panels: - panel_data = db.get_panel_data(experiment_id, panel_config.name) - - if panel_data is not None: - rendered_panels.append( - { - "panel_name": panel_config.name, - "panel_title": panel_config.title, - "panel_type": panel_config.type, - "option": panel_data, - } - ) - else: - rendered_panels.append( - { - "panel_name": panel_config.name, - "panel_title": panel_config.title, - "panel_type": panel_config.type, - "error": "Panel data not found in database. Please reprocess the experiment.", - } - ) - - return {"experiment_id": experiment_id, "dataset_name": dataset_name, "panels": rendered_panels} - - -@router.get("/{panel_name}") -def get_panel(experiment_id: str, panel_name: str, request: Request) -> Dict[str, Any]: - """Get a preprocessed panel with its rendered ECharts option.""" - db = DSTDatabase() - - # Get experiment from database - experiment_data = db.get_experiment(experiment_id) - if experiment_data is None: - raise HTTPException(status_code=404, detail="Experiment not found") - - experiment = ExperimentConfig(**experiment_data) - - # Find panel config - panel_config = None - for panel in experiment.panels: - if panel.name == panel_name: - panel_config = panel - break - - if panel_config is None: - raise HTTPException(status_code=404, detail="Panel not found") - - # Get preprocessed panel data from database - panel_data = db.get_panel_data(experiment_id, panel_name) - - if panel_data is None: - raise HTTPException( - status_code=404, - detail="Panel data not found in database. Please reprocess the experiment.", - ) - - return { - "panel_name": panel_name, - "panel_title": panel_config.title, - "panel_type": panel_config.type, - "option": panel_data, - } - - -@router.delete("/{panel_name}", status_code=204) -def delete_panel( - experiment_id: str, - panel_name: str, - request: Request, - _: None = Depends(require_admin_token), -): - """Delete a panel.""" - db = DSTDatabase() - - # Get experiment from database - experiment_data = db.get_experiment(experiment_id) - if experiment_data is None: - raise HTTPException(status_code=404, detail="Experiment not found") - - experiment = ExperimentConfig(**experiment_data) - - # Find panel - panel_index = None - for i, p in enumerate(experiment.panels): - if p.name == panel_name: - panel_index = i - break - - if panel_index is None: - raise HTTPException(status_code=404, detail="Panel not found") - - # Remove panel from experiment - experiment.panels.pop(panel_index) - - # Update experiment in database - db.store_experiment(experiment.model_dump()) - - # Delete cached panel data - db.delete_panel(experiment_id, panel_name) - - return None diff --git a/dst_dashboard/api/utils.py b/dst_dashboard/api/utils.py deleted file mode 100644 index ae23d6ac..00000000 --- a/dst_dashboard/api/utils.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Utility functions for API routes.""" - -import logging - -from fastapi import HTTPException, Request - -from dst_dashboard.processors.experiment_processor import ExperimentProcessor -from dst_dashboard.processors.panel_processor import PanelProcessor -from dst_dashboard.storage.db import DSTDatabase - -logger = logging.getLogger(__name__) - - -def get_processor(request: Request) -> ExperimentProcessor: - """Build an ExperimentProcessor from app state (DB is source of truth for experiments).""" - config = getattr(request.app.state, "config", None) - if config is None: - raise HTTPException(status_code=500, detail="Config is not initialized") - db = DSTDatabase() - return ExperimentProcessor(config, db) - - -def get_panel_processor(request: Request) -> PanelProcessor: - """Build a PanelProcessor from app state.""" - config = getattr(request.app.state, "config", None) - if config is None: - raise HTTPException(status_code=500, detail="Config is not initialized") - db = DSTDatabase() - return PanelProcessor(config, db) diff --git a/dst_dashboard/auth.py b/dst_dashboard/auth.py deleted file mode 100644 index fb6b2114..00000000 --- a/dst_dashboard/auth.py +++ /dev/null @@ -1,56 +0,0 @@ -"""JWT issuing and verification for admin (write) API access.""" - -import logging -import time - -import jwt -from fastapi import Depends, HTTPException -from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer - -from dst_dashboard.config.constants import INSECURE_DEFAULT_JWT_SECRET, Constants - -logger = logging.getLogger(__name__) - -JWT_ALGORITHM = "HS256" -JWT_SCOPE = "admin" - -_bearer_scheme = HTTPBearer(auto_error=False) - -if str(Constants.DST_JWT_SECRET) == INSECURE_DEFAULT_JWT_SECRET: - logger.warning( - "DST_JWT_SECRET is not set - using an insecure development-only default. " - "Set DST_JWT_SECRET before deploying anywhere real." - ) - - -def create_admin_token(expires_days: int = 1) -> str: - """Generate a fresh admin-scope JWT, signed with DST_JWT_SECRET.""" - now = int(time.time()) - payload = { - "scope": JWT_SCOPE, - "iat": now, - "exp": now + expires_days * 86400, - } - return jwt.encode(payload, str(Constants.DST_JWT_SECRET), algorithm=JWT_ALGORITHM) - - -def require_admin_token( - credentials: HTTPAuthorizationCredentials = Depends(_bearer_scheme), -) -> None: - """FastAPI dependency gating mutating routes behind a valid admin-scope JWT.""" - if credentials is None: - raise HTTPException(status_code=401, detail="Missing bearer token") - - try: - payload = jwt.decode( - credentials.credentials, - str(Constants.DST_JWT_SECRET), - algorithms=[JWT_ALGORITHM], - ) - except jwt.ExpiredSignatureError: - raise HTTPException(status_code=401, detail="Token has expired") - except jwt.InvalidTokenError: - raise HTTPException(status_code=401, detail="Invalid token") - - if payload.get("scope") != JWT_SCOPE: - raise HTTPException(status_code=403, detail="Token does not have admin scope") diff --git a/dst_dashboard/config/__init__.py b/dst_dashboard/config/__init__.py deleted file mode 100644 index 8b137891..00000000 --- a/dst_dashboard/config/__init__.py +++ /dev/null @@ -1 +0,0 @@ - diff --git a/dst_dashboard/config/constants.py b/dst_dashboard/config/constants.py deleted file mode 100644 index 4f6a4c38..00000000 --- a/dst_dashboard/config/constants.py +++ /dev/null @@ -1,36 +0,0 @@ -import os -from enum import StrEnum -from pathlib import Path - -DEFAULT_CONFIG_PATH = Path.home() / ".cache" / "dst_dashboard" / "config.yaml" -DEFAULT_DATA_PATH = Path.home() / ".cache" / "dst_dashboard" / "data" -DEFAULT_MONGO_URI = "mongodb://localhost:27017" -DEFAULT_MONGO_DB_NAME = "dst_dashboard" -# Clearly-marked insecure fallback so it's obvious in logs/code review if a real -# secret was never configured - never rely on this outside local dev. -INSECURE_DEFAULT_JWT_SECRET = "dev-only-insecure-secret-change-me" - - -class Constants(StrEnum): - """Constants used in DST dashboard application.""" - - DST_CONFIG_PATH = os.environ.get( - "DST_CONFIG_PATH", - str(DEFAULT_CONFIG_PATH), - ) - DST_DATA_PATH = os.environ.get( - "DST_DATA_PATH", - str(DEFAULT_DATA_PATH), - ) - DST_MONGO_URI = os.environ.get( - "DST_MONGO_URI", - DEFAULT_MONGO_URI, - ) - DST_MONGO_DB_NAME = os.environ.get( - "DST_MONGO_DB_NAME", - DEFAULT_MONGO_DB_NAME, - ) - DST_JWT_SECRET = os.environ.get( - "DST_JWT_SECRET", - INSECURE_DEFAULT_JWT_SECRET, - ) diff --git a/dst_dashboard/config/data_structures.py b/dst_dashboard/config/data_structures.py deleted file mode 100644 index d6969d5e..00000000 --- a/dst_dashboard/config/data_structures.py +++ /dev/null @@ -1,125 +0,0 @@ -from datetime import datetime -from typing import Any, Dict, List, Literal, Optional - -from pydantic import BaseModel - - -class TimeRange(BaseModel): - """Time range for data queries.""" - - start: datetime - end: datetime - - -class DataSourceConfig(BaseModel): - """Global datasource configuration.""" - - name: str - type: Literal["VictoriaLogs", "Prometheus"] - url: str - - -class DatasetQuery(BaseModel): - """Query configuration for a dataset.""" - - namespace: Optional[str] = None - tracer: Optional[str] = None # For logs: nimlibp2p, kad_dht, waku - pattern: Optional[str] = None # For logs: received, lookup, etc. - expr: Optional[str] = None # For metrics: PromQL expression - step: Optional[str] = None # For metrics: step interval - - -class SchemaField(BaseModel): - """Schema field definition.""" - - name: str - type: Literal["datetime", "string", "float", "integer", "boolean"] - - -class DatasetConfig(BaseModel): - """Dataset configuration - raw data from datasources.""" - - name: str - datasource: str # References global datasource by name - timeRange: TimeRange - query: DatasetQuery - schema: List[SchemaField] - - -class DeriveField(BaseModel): - """Derived field transformation in panel.""" - - name: str - function: str # regex_match, aggregate, etc. - field: str # Source field - pattern: Optional[str] = None # For regex - match: Optional[str] = None # Regex match value - no_match: Optional[str] = None # Regex no-match value - - -class PanelTransform(BaseModel): - """Panel transformation configuration.""" - - derive: Optional[List[DeriveField]] = None - groupBy: Optional[str] = None - value: Optional[str] = None - x: Optional[str] = None - y: Optional[str] = None - top: Optional[int] = None # Top N by average value - firstN: Optional[int] = None # First N items (no sorting) - - -class PanelStyle(BaseModel): - """Panel style configuration.""" - - yLabel: Optional[str] = None - xLabel: Optional[str] = None - yUnit: Optional[Literal["bytes", "bytes/s", "bps", "ms", "seconds", "percent", "number"]] = ( - None # Auto-format units - ) - yMin: Optional[float] = None - yMax: Optional[float] = None - - -class PanelConfig(BaseModel): - name: str - title: str - type: Literal["boxplot", "timeseries", "histogram", "bar", "table"] - dataset: str # References dataset by name - transform: PanelTransform - style: Optional[PanelStyle] = None - echarts_options: Optional[Dict[str, Any]] = ( - None # Override ECharts options (colors, grid, etc.) - ) - publish: bool # Whether to show on UI - - -class ExperimentConfig(BaseModel): - id: Optional[str] = None # Server-assigned on create; ignored if sent by the client - title: str - family: str - description: Optional[str] = None # Experiment description - metadata: Dict[str, Any] # Flexible metadata from 10ksim - datasets: List[DatasetConfig] - panels: List[PanelConfig] - publish: bool # Whether to show on UI - github_repo: Optional[str] = None # GitHub repository URL - github_pr: Optional[str] = None # Pull request URL - docker_image: Optional[str] = None # Docker image reference - date: Optional[str] = None # ISO date string (YYYY-MM-DD) - - -class DashboardFullConfig(BaseModel): - datasources: List[DataSourceConfig] - - def WithValidateDatasources(self) -> "DashboardFullConfig": - if not self.datasources: - raise ValueError("At least one datasource must be defined in the config.") - for datasource in self.datasources: - if not datasource.name: - raise ValueError("Datasource must have a name.") - if not datasource.type: - raise ValueError("Datasource must have a type.") - if not datasource.url: - raise ValueError("Datasource must have a URL.") - return self diff --git a/dst_dashboard/config/utils.py b/dst_dashboard/config/utils.py deleted file mode 100644 index 55002062..00000000 --- a/dst_dashboard/config/utils.py +++ /dev/null @@ -1,13 +0,0 @@ -import yaml - -from dst_dashboard.config.constants import Constants -from dst_dashboard.config.data_structures import DashboardFullConfig - - -def LoadConfig(config_path: str = Constants.DST_CONFIG_PATH) -> DashboardFullConfig: - with open(config_path, "r", encoding="utf-8") as file: - config_yaml = yaml.safe_load(file) - if config_yaml is None: - raise ValueError(f"Config file '{config_path}' is empty") - config = DashboardFullConfig.model_validate(config_yaml) - return config diff --git a/dst_dashboard/k8s/backend.yaml b/dst_dashboard/k8s/backend.yaml deleted file mode 100644 index 4019d83a..00000000 --- a/dst_dashboard/k8s/backend.yaml +++ /dev/null @@ -1,155 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: dst-dashboard-config - namespace: dst-dashboard -data: - config.yaml: | - # DST Explorer Configuration - # Global datasources - defined once, referenced by all experiments - - datasources: - - name: victoria-logs - type: VictoriaLogs - url: http://vlc-victoria-logs-cluster-vlselect.victorialogs:9471/select/logsql/query - - - name: victoria-metrics - type: Prometheus - url: http://vmselect-vmks-victoria-metrics-k8s-stack.vmetrics:8481/select/0/prometheus/api/v1/ ---- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: dst-dashboard-backend - namespace: dst-dashboard -spec: - serviceName: dst-dashboard-backend - replicas: 1 - selector: - matchLabels: - app: dst-dashboard-backend - template: - metadata: - labels: - app: dst-dashboard-backend - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: kubernetes.io/hostname - operator: In - values: - - metal-01.he-eu-hel1.misc.vacdst - tolerations: - - key: "dedicated" - operator: "Equal" - value: "monitoring" - effect: "PreferNoSchedule" - containers: - - name: dst-dashboard-backend - image: mamoutoudiarra/dst-dashboard-backend:v0.5 - imagePullPolicy: IfNotPresent - ports: - - containerPort: 8000 - name: http - env: - - name: DST_MONGO_URI - valueFrom: - secretKeyRef: - name: dst-mongo-dev-credentials - key: uri - - name: DST_JWT_SECRET - valueFrom: - secretKeyRef: - name: dst-dashboard-jwt-secret - key: secret - - name: DST_CONFIG_PATH - value: /app/config/config.yaml - volumeMounts: - - name: config - mountPath: /app/config - readOnly: true - resources: - requests: - memory: "4Gi" - cpu: "4" - limits: - memory: "16Gi" - cpu: "8" - volumes: - - name: config - configMap: - name: dst-dashboard-config ---- -apiVersion: v1 -kind: Service -metadata: - name: dst-dashboard-backend - namespace: dst-dashboard -spec: - selector: - app: dst-dashboard-backend - ports: - - protocol: TCP - port: 8000 - targetPort: 8000 - type: ClusterIP - clusterIP: None ---- -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: dst-dashboard-backend-ingress - namespace: dst-dashboard - annotations: - traefik.ingress.kubernetes.io/router.middlewares: default-redirect-https@kubernetescrd - cert-manager.io/cluster-issuer: letsencrypt-prod -spec: - ingressClassName: traefik - tls: - - hosts: - - api.dashboard.lab.vac.dev - secretName: dashboard-backend-tls - rules: - - host: api.dashboard.lab.vac.dev - http: - paths: - - path: / - pathType: Prefix - backend: - service: - name: dst-dashboard-backend - port: - number: 8000 ---- -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: dst-dashboard-backend-admin-token-ingress - namespace: dst-dashboard - annotations: - # Same host as the main ingress above, but this path additionally requires - # Authentik SSO via forward-auth before Traefik ever proxies to the app - - # this page is the only way to read a write-scoped API token, so it's - # gated at the infra layer rather than by the app itself. - traefik.ingress.kubernetes.io/router.middlewares: default-redirect-https@kubernetescrd,default-authentik-forwardauth@kubernetescrd - cert-manager.io/cluster-issuer: letsencrypt-prod -spec: - ingressClassName: traefik - tls: - - hosts: - - api.dashboard.lab.vac.dev - secretName: dashboard-backend-tls - rules: - - host: api.dashboard.lab.vac.dev - http: - paths: - - path: /admin/token - pathType: Exact - backend: - service: - name: dst-dashboard-backend - port: - number: 8000 diff --git a/dst_dashboard/k8s/frontend.yaml b/dst_dashboard/k8s/frontend.yaml deleted file mode 100644 index ebf1fdcf..00000000 --- a/dst_dashboard/k8s/frontend.yaml +++ /dev/null @@ -1,84 +0,0 @@ -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: dst-dashboard-frontend - namespace: dst-dashboard -spec: - serviceName: dst-dashboard-frontend - replicas: 2 - selector: - matchLabels: - app: dst-dashboard-frontend - template: - metadata: - labels: - app: dst-dashboard-frontend - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: kubernetes.io/hostname - operator: In - values: - - metal-01.he-eu-hel1.misc.vacdst - tolerations: - - key: "dedicated" - operator: "Equal" - value: "monitoring" - effect: "PreferNoSchedule" - containers: - - name: frontend - image: mamoutoudiarra/dst-dashboard-frontend:v0.5 - imagePullPolicy: IfNotPresent - ports: - - containerPort: 80 - name: http - resources: - requests: - memory: "512Mi" - cpu: "500m" - limits: - memory: "2Gi" - cpu: "2" ---- -apiVersion: v1 -kind: Service -metadata: - name: dst-dashboard-frontend - namespace: dst-dashboard -spec: - selector: - app: dst-dashboard-frontend - ports: - - protocol: TCP - port: 80 - targetPort: 80 - type: ClusterIP ---- -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: dst-dashboard-frontend-ingress - namespace: dst-dashboard - annotations: - traefik.ingress.kubernetes.io/router.middlewares: default-redirect-https@kubernetescrd - cert-manager.io/cluster-issuer: letsencrypt-prod -spec: - ingressClassName: traefik - tls: - - hosts: - - dashboard.lab.vac.dev - secretName: dashboard-frontend-tls - rules: - - host: dashboard.lab.vac.dev - http: - paths: - - path: / - pathType: Prefix - backend: - service: - name: dst-dashboard-frontend - port: - number: 80 diff --git a/dst_dashboard/k8s/mongodb-dev.yaml b/dst_dashboard/k8s/mongodb-dev.yaml deleted file mode 100644 index 1d83f317..00000000 --- a/dst_dashboard/k8s/mongodb-dev.yaml +++ /dev/null @@ -1,79 +0,0 @@ -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: dst-mongo-dev-pvc - namespace: dst-dashboard -spec: - accessModes: - - ReadWriteOnce - storageClassName: longhorn-vaclab - resources: - requests: - storage: 40Gi ---- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: mongodb-dev - namespace: dst-dashboard -spec: - serviceName: mongodb-dev - replicas: 1 - selector: - matchLabels: - app: mongodb-dev - template: - metadata: - labels: - app: mongodb-dev - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: kubernetes.io/hostname - operator: In - values: - - metal-01.he-eu-hel1.misc.vacdst - tolerations: - - key: "dedicated" - operator: "Equal" - value: "monitoring" - effect: "PreferNoSchedule" - containers: - - name: mongodb-dev - image: mongo:8 - imagePullPolicy: IfNotPresent - ports: - - containerPort: 27017 - name: mongodb-dev - protocol: TCP - env: - - name: MONGO_INITDB_ROOT_USERNAME - value: "root" - - name: MONGO_INITDB_ROOT_PASSWORD - value: "" #password - volumeMounts: - - name: mongo-data-dev - mountPath: /data/db - volumes: - - name: mongo-data-dev - persistentVolumeClaim: - claimName: dst-mongo-dev-pvc - ---- -apiVersion: v1 -kind: Service -metadata: - name: mongodb-dev - namespace: dst-dashboard -spec: - type: LoadBalancer - selector: - app: mongodb-dev - ports: - - name: mongodb-dev - protocol: TCP - port: 8021 - targetPort: 27017 diff --git a/dst_dashboard/k8s/mongodb-prod.yaml b/dst_dashboard/k8s/mongodb-prod.yaml deleted file mode 100644 index d4d372ae..00000000 --- a/dst_dashboard/k8s/mongodb-prod.yaml +++ /dev/null @@ -1,79 +0,0 @@ -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: dst-mongo-prod-pvc - namespace: dst-dashboard -spec: - accessModes: - - ReadWriteOnce - storageClassName: longhorn-vaclab - resources: - requests: - storage: 100Gi ---- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: mongodb-prod - namespace: dst-dashboard -spec: - serviceName: mongodb-prod - replicas: 1 - selector: - matchLabels: - app: mongodb-prod - template: - metadata: - labels: - app: mongodb-prod - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: kubernetes.io/hostname - operator: In - values: - - metal-01.he-eu-hel1.misc.vacdst - tolerations: - - key: "dedicated" - operator: "Equal" - value: "monitoring" - effect: "PreferNoSchedule" - containers: - - name: mongodb-prod - image: mongo:8 - imagePullPolicy: IfNotPresent - ports: - - containerPort: 27017 - name: mongodb-prod - protocol: TCP - env: - - name: MONGO_INITDB_ROOT_USERNAME - value: "root" - - name: MONGO_INITDB_ROOT_PASSWORD - value: "" #password - volumeMounts: - - name: mongo-data-prod - mountPath: /data/db - volumes: - - name: mongo-data-prod - persistentVolumeClaim: - claimName: dst-mongo-prod-pvc - ---- -apiVersion: v1 -kind: Service -metadata: - name: mongodb-prod - namespace: dst-dashboard -spec: - type: ClusterIP - selector: - app: mongodb-prod - ports: - - name: mongodb-prod - protocol: TCP - port: 8021 - targetPort: 27017 diff --git a/dst_dashboard/main.py b/dst_dashboard/main.py deleted file mode 100644 index d9d925c7..00000000 --- a/dst_dashboard/main.py +++ /dev/null @@ -1,86 +0,0 @@ -import logging -import sys - -from fastapi import FastAPI -from fastapi.middleware.cors import CORSMiddleware - -from dst_dashboard.api import admin, datasets, datasources, experiments, panels -from dst_dashboard.config.utils import LoadConfig -from dst_dashboard.storage.db import DSTDatabase - -logging.basicConfig( - level=logging.DEBUG, - format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", - handlers=[logging.StreamHandler()], -) -logger = logging.getLogger(__name__) - -app = FastAPI( - title="DST Dashboard API", - description="REST API for DST Dashboard", - version="0.1.0", - docs_url="/api/docs", - redoc_url="/api/redoc", - openapi_url="/api/openapi.json", -) - - -@app.on_event("startup") -def on_startup(): - """ - Initialize application on startup. - - Experiments are managed exclusively through the API and live only in the - database - config.yaml only defines datasources, so startup just loads - and stores those. No experiment processing happens at boot. - """ - try: - logger.info("Starting DST Dashboard initialization...") - - config = LoadConfig().WithValidateDatasources() - logger.info(f"Loaded {len(config.datasources)} datasources from config.yaml") - - app.state.config = config - app.state.datasources = config.datasources - - db = DSTDatabase() - db.insert_datasource_list(config.datasources) - logger.info(f"Stored {len(config.datasources)} datasources") - - logger.info("DST Dashboard initialization completed") - - except Exception as e: - logger.error(f"Failed to initialize DST Dashboard: {e}", exc_info=True) - sys.exit(1) - - -# Enable CORS for frontend -app.add_middleware( - CORSMiddleware, - allow_origins=["*"], - allow_credentials=True, - allow_methods=["*"], - allow_headers=["*"], -) - -# Include routers -app.include_router(admin.router) -app.include_router(experiments.router) -app.include_router(datasources.router) -app.include_router(datasets.router) -app.include_router(panels.router) - - -@app.get("/") -def root(): - return { - "service": "DST Dashboard API", - "version": "0.1.0", - "docs": "/api/docs", - } - - -if __name__ == "__main__": - import uvicorn - - uvicorn.run(app, host="0.0.0.0", port=8000) diff --git a/dst_dashboard/processors/__init__.py b/dst_dashboard/processors/__init__.py deleted file mode 100644 index 8b137891..00000000 --- a/dst_dashboard/processors/__init__.py +++ /dev/null @@ -1 +0,0 @@ - diff --git a/dst_dashboard/processors/dataset_processor.py b/dst_dashboard/processors/dataset_processor.py deleted file mode 100644 index b9e13d23..00000000 --- a/dst_dashboard/processors/dataset_processor.py +++ /dev/null @@ -1,374 +0,0 @@ -"""Dataset processor - base class for fetching and processing datasets.""" - -import logging -import threading -from typing import Any, Dict, List, Optional - -import pandas as pd -from result import Err, Ok - -from dst_dashboard.config.data_structures import ( - DashboardFullConfig, - DatasetConfig, - DataSourceConfig, - ExperimentConfig, -) -from dst_dashboard.storage.db import DSTDatabase -from src.analysis.mesh_analysis.analyzers.data_puller import DataPuller -from src.analysis.mesh_analysis.readers.tracers.message_tracer import MessageTracer -from src.analysis.metrics import scrape_utils - -logger = logging.getLogger(__name__) - -_victorialogs_log_init_lock = threading.Lock() -_victorialogs_log_initialized = False - - -def _ensure_victorialogs_logging_initialized(): - """One-time (per-process) init of the log queue DataPuller's worker processes attach to. - - log_utils.apply_config() mutates global logging state (clears every - logger's handlers, restarts a QueueListener), so it must only ever run - once - concurrent dataset fetches (now run via a thread pool) would - otherwise race on that global state. - """ - global _victorialogs_log_initialized - if _victorialogs_log_initialized: - return - with _victorialogs_log_init_lock: - if _victorialogs_log_initialized: - return - from src.analysis.utils.log_utils import Config, apply_config - - try: - config = Config( - logger_name="data_puller", - level=logging.INFO, - fmt="%(asctime)s - %(name)s - %(levelname)s - %(message)s", - handlers=[], - tz_offset_hours=0, - ) - apply_config(config) - logger.debug("Log queue initialized for DataPuller") - except Exception as e: - logger.debug(f"Log queue initialization skipped: {e}") - _victorialogs_log_initialized = True - - -class DatasetProcessor: - """ - Base processor for datasets - fetches data using DataPuller or scrape_utils. - - This is the base class in the processor hierarchy: - ExperimentProcessor -> PanelProcessor -> DatasetProcessor - """ - - def __init__(self, config: DashboardFullConfig, db: DSTDatabase): - self.config = config - self.db = db - self._datasources = {ds.name: ds for ds in config.datasources} - - def _get_datasource(self, datasource_name: str) -> Optional[DataSourceConfig]: - """Get datasource configuration by name.""" - return self._datasources.get(datasource_name) - - def _get_experiment(self, experiment_id: str) -> Optional[ExperimentConfig]: - """Get experiment configuration by ID. Experiments live only in the database.""" - experiment_data = self.db.get_experiment(experiment_id) - if experiment_data is None: - return None - return ExperimentConfig(**experiment_data) - - def _apply_schema( - self, data_rows: List[Dict[str, Any]], dataset_config: DatasetConfig - ) -> List[Dict[str, Any]]: - """Apply schema to data rows - filter fields and apply type conversions.""" - if not dataset_config.schema: - return data_rows - - schema_fields = {field.name: field.type for field in dataset_config.schema} - filtered_rows = [] - - for row in data_rows: - filtered_row = {} - for field_name, field_type in schema_fields.items(): - if field_name not in row: - continue - - value = row[field_name] - - # Apply type conversions - try: - if field_type == "datetime": - if not isinstance(value, pd.Timestamp): - value = pd.to_datetime(value) - elif field_type == "float": - value = float(value) - elif field_type == "int": - value = int(value) - elif field_type == "string": - value = str(value) - - filtered_row[field_name] = value - except (ValueError, TypeError) as e: - logger.warning(f"Failed to convert field '{field_name}' to {field_type}: {e}") - continue - - if filtered_row: # Only add if we have at least some fields - filtered_rows.append(filtered_row) - - return filtered_rows - - def fetch_dataset( - self, experiment_id: str, dataset_config: DatasetConfig - ) -> List[Dict[str, Any]]: - """Fetch dataset using the appropriate fetcher based on datasource type.""" - datasource = self._get_datasource(dataset_config.datasource) - if not datasource: - raise ValueError( - f"Datasource '{dataset_config.datasource}' not found for dataset '{dataset_config.name}'" - ) - - experiment = self._get_experiment(experiment_id) - if not experiment: - raise ValueError(f"Experiment '{experiment_id}' not found") - - try: - if datasource.type == "VictoriaLogs": - return self._fetch_from_victorialogs(dataset_config, datasource, experiment) - elif datasource.type == "Prometheus": - return self._fetch_from_prometheus(dataset_config, datasource, experiment) - else: - raise ValueError(f"Unsupported datasource type: {datasource.type}") - except Exception as e: - logger.error( - f"Failed to fetch dataset '{dataset_config.name}' from {datasource.type}: {e}" - ) - raise - - def _get_tracer_for_dataset(self, dataset_config: DatasetConfig): - """Create the appropriate tracer based on dataset configuration.""" - tracer_type = dataset_config.query.tracer - pattern = dataset_config.query.pattern - - # Extract kubernetes fields from schema that need to be fetched as extra_fields - extra_fields = [] - for field in dataset_config.schema: - # Map common schema field names to kubernetes field names - if field.name == "pod_name": - extra_fields.append("kubernetes.pod_name") - elif field.name == "namespace": - extra_fields.append("kubernetes.namespace") - elif field.name == "container_name": - extra_fields.append("kubernetes.container_name") - elif field.name.startswith("kubernetes."): - extra_fields.append(field.name) - - if tracer_type == "nimlibp2p": - from src.analysis.mesh_analysis.readers.tracers.nimlibp2p_tracer import Nimlibp2pTracer - - tracer = Nimlibp2pTracer() - - if extra_fields: - tracer = tracer.with_extra_fields(extra_fields) - - # Use specific pattern if provided - if pattern == "received": - tracer = tracer.with_received_pattern_group() - elif pattern == "sent": - tracer = tracer.with_sent_pattern_group() - else: - # Default to received for message delay analysis - tracer = tracer.with_received_pattern_group() - - return tracer - else: - # Fallback to generic MessageTracer - - tracer = MessageTracer() - if pattern: - tracer.with_wildcard_pattern() - else: - tracer.with_wildcard_pattern() - return tracer - - def _fetch_from_victorialogs( - self, - dataset_config: DatasetConfig, - datasource: DataSourceConfig, - experiment: ExperimentConfig, - ) -> List[Dict[str, Any]]: - """Fetch logs from VictoriaLogs using DataPuller.""" - logger.info(f"Fetching dataset '{dataset_config.name}' from VictoriaLogs") - - # Get statefulsets and nodes from experiment metadata - stateful_sets = experiment.metadata.get("statefulsets", []) - nodes_per_ss = experiment.metadata.get("nodes", []) - - if not stateful_sets or not nodes_per_ss: - logger.warning( - f"Experiment '{experiment.id}' missing statefulsets or nodes in metadata - returning empty dataset" - ) - return [] - - # Build kwargs for DataPuller - kwargs = { - "url": datasource.url, - "start_time": dataset_config.timeRange.start.isoformat(), - "end_time": dataset_config.timeRange.end.isoformat(), - "extra_fields": [field.name for field in dataset_config.schema], - "namespace": dataset_config.query.namespace or experiment.metadata.get("namespace", ""), - "stateful_sets": stateful_sets, - "nodes_per_statefulset": nodes_per_ss, - "container_name": experiment.metadata.get("container_name", ""), - } - - # Get tracer based on configuration - tracer = self._get_tracer_for_dataset(dataset_config) - - _ensure_victorialogs_logging_initialized() - - # Initialize DataPuller - data_puller = DataPuller().with_kwargs(kwargs).with_source_type("victoria") - - try: - logger.debug(f"DataPuller kwargs: {kwargs}") - logger.debug( - f"Fetching with stateful_sets={stateful_sets}, nodes_per_ss={nodes_per_ss}" - ) - - # Fetch dataframes from VictoriaLogs - results = data_puller.get_all_node_dataframes(tracer, stateful_sets, nodes_per_ss) - - logger.debug(f"DataPuller returned {len(results)} result dicts") - - # Convert results to standard format - data_rows = [] - for idx, result_dict in enumerate(results): - logger.debug(f"Processing result_dict {idx}: {list(result_dict.keys())}") - for pattern_name, df_list in result_dict.items(): - logger.debug(f"Pattern '{pattern_name}': {len(df_list)} dataframes") - for df_idx, df in enumerate(df_list): - if isinstance(df, pd.DataFrame): - if not df.empty: - logger.debug( - f"DataFrame {df_idx} shape: {df.shape}, columns: {list(df.columns)}" - ) - # Convert DataFrame to list of dicts - df_dict = df.reset_index().to_dict(orient="records") - data_rows.extend(df_dict) - else: - logger.debug(f"DataFrame {df_idx} is empty") - elif isinstance(df, list): - # Handle list format directly - logger.debug(f"List {df_idx} has {len(df)} items") - if df: # If list is not empty - # Check if items are already dicts - if isinstance(df[0], dict): - data_rows.extend(df) - else: - logger.warning(f"List contains non-dict items: {type(df[0])}") - else: - logger.warning(f"Result is not a DataFrame or list: {type(df)}") - - logger.info(f"Converted {len(data_rows)} raw rows from VictoriaLogs") - - # Normalize kubernetes.* field names (e.g., kubernetes.pod_name -> pod_name) - normalized_rows = [] - for row in data_rows: - normalized_row = {} - for key, value in row.items(): - if key.startswith("kubernetes."): - # Remove 'kubernetes.' prefix - new_key = key.replace("kubernetes.", "") - normalized_row[new_key] = value - else: - normalized_row[key] = value - normalized_rows.append(normalized_row) - - logger.debug(f"Normalized {len(normalized_rows)} rows (renamed kubernetes.* fields)") - - # Apply schema filtering and type conversions - filtered_rows = self._apply_schema(normalized_rows, dataset_config) - - logger.info( - f"Fetched {len(data_rows)} rows from VictoriaLogs, " - f"{len(filtered_rows)} rows after schema filtering for dataset '{dataset_config.name}'" - ) - return filtered_rows - - except Exception as e: - logger.error(f"Error fetching from VictoriaLogs: {e}", exc_info=True) - return [] - - def _fetch_from_prometheus( - self, - dataset_config: DatasetConfig, - datasource: DataSourceConfig, - experiment: ExperimentConfig, - ) -> List[Dict[str, Any]]: - """Fetch metrics from Prometheus using direct query.""" - logger.info(f"Fetching dataset '{dataset_config.name}' from Prometheus") - - if not dataset_config.query.expr: - raise ValueError( - f"Dataset '{dataset_config.name}' missing required 'expr' for Prometheus query" - ) - - # Parse step to integer (remove 's' suffix if present) - step = dataset_config.query.step or "15s" - step_value = int(step.rstrip("s")) - - # Create PromQL query URL - promql_url = scrape_utils.create_promql( - datasource.url, - dataset_config.query.expr, - dataset_config.timeRange.start, - dataset_config.timeRange.end, - step_value, - ) - - logger.debug(f"Prometheus query URL: {promql_url}") - - try: - # Execute query - result = scrape_utils.get_query_data(promql_url) - - match result: - case Ok(data): - logger.info( - f"Successfully fetched data from Prometheus for dataset '{dataset_config.name}'" - ) - - # Convert Prometheus response to standard format - data_rows = [] - for series in data["data"]["result"]: - metric_labels = series.get("metric", {}) - values = series.get("values", []) - - for timestamp, value in values: - # Build row with timestamp, value, and all metric labels - row = { - "timestamp": pd.to_datetime(timestamp, unit="s"), - "value": float(value), - } - # Add all metric labels as potential fields - row.update(metric_labels) - data_rows.append(row) - - # Apply schema filtering and type conversions - filtered_rows = self._apply_schema(data_rows, dataset_config) - - logger.info( - f"Fetched {len(data_rows)} rows from Prometheus, " - f"{len(filtered_rows)} rows after schema filtering for dataset '{dataset_config.name}'" - ) - return filtered_rows - - case Err(error): - logger.error(f"Prometheus query error: {error}") - return [] - - except Exception as e: - logger.error(f"Error fetching from Prometheus: {e}") - return [] diff --git a/dst_dashboard/processors/experiment_processor.py b/dst_dashboard/processors/experiment_processor.py deleted file mode 100644 index e5f278a4..00000000 --- a/dst_dashboard/processors/experiment_processor.py +++ /dev/null @@ -1,131 +0,0 @@ -"""Experiment processor - processes complete experiments with datasets and panels.""" - -import logging -from concurrent.futures import ThreadPoolExecutor, as_completed - -from bson import ObjectId - -from dst_dashboard.config.data_structures import ( - DashboardFullConfig, - DatasetConfig, - ExperimentConfig, -) -from dst_dashboard.processors.panel_processor import PanelProcessor -from dst_dashboard.storage.db import DSTDatabase - -logger = logging.getLogger(__name__) - - -class ExperimentProcessor(PanelProcessor): - """ - Experiment processor - top-level processor. - """ - - def __init__(self, config: DashboardFullConfig, db: DSTDatabase): - super().__init__(config, db) - - def _ensure_experiment_id(self, experiment: ExperimentConfig) -> str: - """Ensure experiment has a valid ID, generating one (Mongo ObjectId-style) if missing.""" - if not experiment.id or experiment.id.strip() == "": - generated_id = str(ObjectId()) - logger.info(f"Generated ID for experiment '{experiment.title}': {generated_id}") - experiment.id = generated_id - - return experiment.id - - def process_dataset(self, experiment_id: str, dataset_config: DatasetConfig) -> bool: - """Process a single dataset - fetch and store if needed. Returns True on success.""" - logger.info(f"Processing dataset: {dataset_config.name}") - - existing_data = self.db.get_dataset(experiment_id, dataset_config.name) - if existing_data is not None: - logger.info( - f"Dataset '{dataset_config.name}' already exists with {len(existing_data)} rows, skipping fetch" - ) - return True - - # Fetch and store dataset - try: - logger.info( - f"Fetching dataset '{dataset_config.name}' from {dataset_config.datasource}" - ) - data = self.fetch_dataset(experiment_id, dataset_config) - - if data: - self.db.store_dataset(experiment_id, dataset_config.name, data) - logger.info(f"Stored {len(data)} rows for dataset '{dataset_config.name}'") - return True - else: - logger.warning(f"No data fetched for dataset '{dataset_config.name}'") - # Store empty dataset to mark it as attempted - self.db.store_dataset(experiment_id, dataset_config.name, []) - return False - - except Exception as e: - logger.error(f"Failed to fetch dataset '{dataset_config.name}': {e}") - # Store empty dataset to mark it as processed but failed - self.db.store_dataset(experiment_id, dataset_config.name, []) - return False - - def process_experiment_datasets( - self, experiment: ExperimentConfig, max_workers: int = 4 - ) -> int: - """Process all datasets for an experiment concurrently (fetches are I/O-bound). Returns the number processed successfully.""" - if not experiment.datasets: - return 0 - - if len(experiment.datasets) == 1 or max_workers <= 1: - return sum( - self.process_dataset(experiment.id, dataset_config) - for dataset_config in experiment.datasets - ) - - success_count = 0 - with ThreadPoolExecutor(max_workers=max_workers) as executor: - futures = { - executor.submit(self.process_dataset, experiment.id, dataset_config): dataset_config - for dataset_config in experiment.datasets - } - for future in as_completed(futures): - dataset_config = futures[future] - try: - if future.result(): - success_count += 1 - except Exception: - logger.error( - f"Dataset '{dataset_config.name}' processing raised unexpectedly", - exc_info=True, - ) - - return success_count - - def process_experiment(self, experiment: ExperimentConfig) -> str: - """Process a complete experiment - store it, fetch datasets, store panels. Returns the experiment ID.""" - experiment_id = self._ensure_experiment_id(experiment) - - logger.info(f"Processing experiment: {experiment_id} - {experiment.title}") - - # 1. Store experiment in database - experiment_dict = experiment.model_dump() - existing_exp = self.db.get_experiment(experiment_id) - - if existing_exp: - logger.info(f"Experiment '{experiment_id}' already exists in database, updating...") - else: - logger.info(f"Storing new experiment '{experiment_id}' in database") - - self.db.store_experiment(experiment_dict) - - # 2. Process datasets (datasets depend on datasources) - dataset_count = self.process_experiment_datasets(experiment) - logger.info( - f"Processed {dataset_count}/{len(experiment.datasets)} datasets for experiment '{experiment_id}'" - ) - - # 3. Process panels (panels depend on datasets) - panel_count = self.process_experiment_panels(experiment) - logger.info( - f"Processed {panel_count}/{len(experiment.panels)} panels for experiment '{experiment_id}'" - ) - - return experiment_id diff --git a/dst_dashboard/processors/panel_processor.py b/dst_dashboard/processors/panel_processor.py deleted file mode 100644 index 0b15a157..00000000 --- a/dst_dashboard/processors/panel_processor.py +++ /dev/null @@ -1,500 +0,0 @@ -"""Panel processor - processes panel configurations and transformations.""" - -import logging -import re -from collections import defaultdict -from concurrent.futures import ThreadPoolExecutor, as_completed -from copy import deepcopy -from typing import Any, Dict, List - -from dst_dashboard.config.data_structures import DashboardFullConfig, ExperimentConfig, PanelConfig -from dst_dashboard.processors.dataset_processor import DatasetProcessor -from dst_dashboard.storage.db import DSTDatabase - -logger = logging.getLogger(__name__) - - -def deep_merge(base: Dict, override: Dict) -> Dict: - """Deep merge two dictionaries. Override values take precedence.""" - result = deepcopy(base) - - for key, value in override.items(): - if key in result and isinstance(result[key], dict) and isinstance(value, dict): - result[key] = deep_merge(result[key], value) - else: - result[key] = deepcopy(value) - - return result - - -class PanelProcessor(DatasetProcessor): - """ - Panel processor - inherits from DatasetProcessor. - - Processes panel configurations, transforms data, and generates ECharts-ready specs. - Hierarchy: ExperimentProcessor -> PanelProcessor -> DatasetProcessor - """ - - def __init__(self, config: DashboardFullConfig, db: DSTDatabase): - super().__init__(config, db) - - def process_panel(self, experiment_id: str, panel_config: PanelConfig) -> bool: - """Process and store panel ECharts options. Returns True on success.""" - logger.info(f"Processing panel: {panel_config.name}") - - # Verify that the dataset for this panel exists - if not self.db.dataset_exists(experiment_id, panel_config.dataset): - logger.warning( - f"Panel '{panel_config.name}' references non-existent dataset '{panel_config.dataset}', skipping" - ) - return False - - try: - # Generate and store ECharts options - echarts_option = self.transform_panel_data(experiment_id, panel_config) - self.db.store_panel_data(experiment_id, panel_config.name, echarts_option) - logger.info(f"Panel '{panel_config.name}' processed and stored successfully") - return True - except Exception as e: - logger.error(f"Failed to process panel '{panel_config.name}': {e}", exc_info=True) - return False - - def process_experiment_panels(self, experiment: ExperimentConfig, max_workers: int = 4) -> int: - """Process all panels for an experiment concurrently. Returns the number processed successfully.""" - if not experiment.panels: - return 0 - - if len(experiment.panels) == 1 or max_workers <= 1: - return sum( - self.process_panel(experiment.id, panel_config) - for panel_config in experiment.panels - ) - - success_count = 0 - with ThreadPoolExecutor(max_workers=max_workers) as executor: - futures = { - executor.submit(self.process_panel, experiment.id, panel_config): panel_config - for panel_config in experiment.panels - } - for future in as_completed(futures): - panel_config = futures[future] - try: - if future.result(): - success_count += 1 - except Exception: - logger.error( - f"Panel '{panel_config.name}' processing raised unexpectedly", exc_info=True - ) - - return success_count - - def _apply_derive_transformations( - self, data: List[Dict[str, Any]], panel_config: PanelConfig - ) -> List[Dict[str, Any]]: - """Apply derive transformations to create new fields.""" - if not panel_config.transform or not panel_config.transform.derive: - return data - - result = [] - for row in data: - new_row = row.copy() - - for derive in panel_config.transform.derive: - if derive.function == "regex_match": - # Apply regex pattern to field - field_value = str(row.get(derive.field, "")) - if derive.pattern and re.search(derive.pattern, field_value): - new_row[derive.name] = derive.match or "match" - else: - new_row[derive.name] = derive.no_match or "no_match" - else: - logger.warning(f"Unknown derive function: {derive.function}") - - result.append(new_row) - - return result - - def _transform_to_boxplot( - self, data: List[Dict[str, Any]], panel_config: PanelConfig - ) -> Dict[str, Any]: - """Transform data for ECharts boxplot visualization.""" - transform = panel_config.transform - group_by = transform.groupBy - value_field = transform.value - - if not group_by or not value_field: - raise ValueError("Boxplot requires 'groupBy' and 'value' in transform") - - # Group data by the groupBy field - grouped_data = defaultdict(list) - for row in data: - group_key = row.get(group_by, "unknown") - value = row.get(value_field) - if value is not None: - grouped_data[group_key].append(float(value)) - - # Apply top filter if specified - sort by average value - if transform.top: - # Calculate average for each group - group_stats = [] - for group_name, values in grouped_data.items(): - avg_val = sum(values) / len(values) if values else 0 - group_stats.append((group_name, avg_val, values)) - - # Sort by average value (descending) and take top N - group_stats.sort(key=lambda x: x[1], reverse=True) - top_groups = group_stats[: transform.top] - - # Rebuild grouped_data with only top groups - grouped_data = {name: values for name, _, values in top_groups} - - # Sort categories and calculate boxplot stats - categories = sorted(grouped_data.keys()) - boxplot_data = [] - - for category in categories: - values = sorted(grouped_data[category]) - if not values: - continue - - n = len(values) - min_val = values[0] - max_val = values[-1] - median = values[n // 2] if n % 2 == 1 else (values[n // 2 - 1] + values[n // 2]) / 2 - q1 = values[n // 4] - q3 = values[3 * n // 4] - - # ECharts boxplot format: [min, Q1, median, Q3, max] - boxplot_data.append([min_val, q1, median, q3, max_val]) - - # Build ECharts option - option = { - "backgroundColor": "transparent", - "title": { - "text": panel_config.title, - "left": "center", - "textStyle": {"color": "#152521"}, - }, - "tooltip": {"trigger": "item", "axisPointer": {"type": "shadow"}}, - "toolbox": { - "feature": { - "dataZoom": { - "yAxisIndex": "none", - "title": {"zoom": "Area Zoom", "back": "Reset"}, - }, - "restore": {"title": "Restore"}, - "saveAsImage": {"title": "Save"}, - }, - "iconStyle": {"borderColor": "#848e88"}, - "emphasis": {"iconStyle": {"borderColor": "#00d4ff"}}, - }, - "dataZoom": [ - {"type": "inside", "xAxisIndex": 0, "filterMode": "none", "disabled": False} - ], - "grid": {"left": "10%", "right": "10%", "bottom": "15%"}, - "xAxis": { - "type": "category", - "data": categories, - "boundaryGap": True, - "splitArea": {"show": False}, - "axisLabel": {"color": "#475651"}, - "axisLine": {"lineStyle": {"color": "#b8bdb8"}}, - }, - "yAxis": { - "type": "value", - "splitArea": {"show": False}, - "axisLabel": {"color": "#475651"}, - "axisLine": {"lineStyle": {"color": "#b8bdb8"}}, - "splitLine": {"lineStyle": {"color": "#eceee4"}}, - }, - "series": [ - { - "name": panel_config.title, - "type": "boxplot", - "data": boxplot_data, - "itemStyle": {"borderColor": "#00d4ff", "borderWidth": 1.5}, - } - ], - } - - # Add optional style configurations - if panel_config.style: - if panel_config.style.xLabel: - option["xAxis"]["name"] = panel_config.style.xLabel - if panel_config.style.yLabel: - option["yAxis"]["name"] = panel_config.style.yLabel - if panel_config.style.yMin is not None: - option["yAxis"]["min"] = panel_config.style.yMin - if panel_config.style.yMax is not None: - option["yAxis"]["max"] = panel_config.style.yMax - - # Merge user-provided ECharts options (if any) - if panel_config.echarts_options: - option = deep_merge(option, panel_config.echarts_options) - - return option - - def _transform_to_timeseries( - self, data: List[Dict[str, Any]], panel_config: PanelConfig - ) -> Dict[str, Any]: - """Transform data for ECharts timeseries (line chart) visualization.""" - transform = panel_config.transform - x_field = transform.x - y_field = transform.y - - if not x_field or not y_field: - raise ValueError("Timeseries requires 'x' and 'y' in transform") - - # Group by a categorical field if groupBy is specified (for multi-series) - if transform.groupBy: - grouped_series = defaultdict(list) - - for row in data: - group_key = row.get(transform.groupBy, "default") - x_val = row.get(x_field) - y_val = row.get(y_field) - - if x_val is not None and y_val is not None: - # Convert datetime to ISO string for JSON serialization - x_val_serialized = ( - x_val.isoformat() if hasattr(x_val, "isoformat") else str(x_val) - ) - grouped_series[group_key].append( - {"timestamp": x_val_serialized, "value": float(y_val)} - ) - - # Apply top filter if specified (limit number of series) - if transform.top: - # Sort by average value to get most significant series - series_stats = [] - for group_name, points in grouped_series.items(): - avg_val = sum(p["value"] for p in points) / len(points) if points else 0 - series_stats.append((group_name, avg_val, points)) - - # Sort by average value (descending) and take top N - series_stats.sort(key=lambda x: x[1], reverse=True) - top_series = dict( - (name, points) for name, _, points in series_stats[: transform.top] - ) - grouped_series = top_series - elif transform.firstN: - # Take first N series by name (no sorting by value) - sorted_names = sorted(grouped_series.keys())[: transform.firstN] - grouped_series = {name: grouped_series[name] for name in sorted_names} - - # Professional color palette - distinct and readable - colors = [ - "#5470c6", - "#91cc75", - "#fac858", - "#ee6666", - "#73c0de", - "#3ba272", - "#fc8452", - "#9a60b4", - "#ea7ccc", - "#d4a5a5", - ] - - # Build series for each group - series = [] - for idx, (group_name, points) in enumerate(sorted(grouped_series.items())): - # Sort points by timestamp to ensure proper line drawing - sorted_points = sorted(points, key=lambda p: p["timestamp"]) - - # Create [timestamp, value] pairs for ECharts - properly ordered - data_pairs = [[p["timestamp"], p["value"]] for p in sorted_points] - - color = colors[idx % len(colors)] - series.append( - { - "name": str(group_name), - "type": "line", - "data": data_pairs, - "smooth": False, - "sampling": "lttb", - "symbol": "none", - "connectNulls": False, # Don't connect gaps - "lineStyle": {"width": 1.5, "color": color}, - "emphasis": {"focus": "series", "lineStyle": {"width": 2.5}}, - } - ) - - else: - # Single series - x_values = [] - y_values = [] - - for row in data: - x_val = row.get(x_field) - y_val = row.get(y_field) - - if x_val is not None and y_val is not None: - # Convert datetime to ISO string for JSON serialization - x_val_serialized = ( - x_val.isoformat() if hasattr(x_val, "isoformat") else str(x_val) - ) - x_values.append(x_val_serialized) - y_values.append(float(y_val)) - - series = [ - { - "name": panel_config.title, - "type": "line", - "data": y_values, - "smooth": True, - "sampling": "lttb", - "symbol": "none", - "lineStyle": {"width": 2, "color": "#00d4ff"}, - "areaStyle": {"color": "#00d4ff", "opacity": 0.15}, - } - ] - - # Build ECharts option - option = { - "backgroundColor": "transparent", - "title": { - "text": panel_config.title, - "left": "center", - "top": 10, - "textStyle": {"color": "#152521", "fontSize": 16, "fontWeight": "normal"}, - }, - "tooltip": { - "trigger": "axis", - "axisPointer": { - "type": "line", - "lineStyle": {"color": "#b8bdb8", "type": "dashed"}, - }, - "backgroundColor": "rgba(245, 245, 239, 0.95)", - "borderColor": "#dbddd7", - "borderWidth": 1, - "textStyle": {"color": "#152521", "fontSize": 12}, - "confine": True, - }, - "legend": { - "type": "scroll", - "data": [s["name"] for s in series], - "bottom": 5, - "left": "center", - "textStyle": {"color": "#475651", "fontSize": 11}, - "pageIconColor": "#5470c6", - "pageIconInactiveColor": "#dbddd7", - "pageTextStyle": {"color": "#475651"}, - "icon": "roundRect", - }, - "toolbox": { - "show": True, - "feature": { - "dataZoom": { - "yAxisIndex": "none", - "title": {"zoom": "Area Zoom", "back": "Reset"}, - }, - "restore": {"title": "Restore"}, - "saveAsImage": {"title": "Save"}, - }, - "iconStyle": {"borderColor": "#848e88"}, - "emphasis": {"iconStyle": {"borderColor": "#5470c6"}}, - "top": 10, - "right": 20, - }, - "dataZoom": [{"type": "inside", "xAxisIndex": 0, "filterMode": "none"}], - "grid": { - "left": "3%", - "right": "4%", - "bottom": "50px", - "top": "60px", - "containLabel": True, - }, - "xAxis": { - "type": "time", - "boundaryGap": False, - "axisLabel": {"color": "#475651", "fontSize": 11, "formatter": "{HH}:{mm}"}, - "axisLine": {"lineStyle": {"color": "#b8bdb8"}}, - "axisTick": {"show": False}, - "splitLine": {"show": False}, - }, - "yAxis": { - "type": "value", - "axisLabel": {"color": "#475651", "fontSize": 11}, - "axisLine": {"show": False}, - "axisTick": {"show": False}, - "splitLine": {"lineStyle": {"color": "#eceee4", "type": "solid"}}, - }, - "series": series, - } - - # Add optional style configurations - if panel_config.style: - if panel_config.style.xLabel: - option["xAxis"]["name"] = panel_config.style.xLabel - option["xAxis"]["nameTextStyle"] = {"color": "#475651", "fontSize": 11} - if panel_config.style.yLabel: - option["yAxis"]["name"] = panel_config.style.yLabel - option["yAxis"]["nameTextStyle"] = {"color": "#475651", "fontSize": 11} - if panel_config.style.yMin is not None: - option["yAxis"]["min"] = panel_config.style.yMin - if panel_config.style.yMax is not None: - option["yAxis"]["max"] = panel_config.style.yMax - - # Add auto-formatting based on unit type - if panel_config.style.yUnit in ["bytes", "bytes/s", "bps"]: - option["yAxis"]["axisLabel"]["formatter"] = "__BYTES_FORMATTER__" - option["tooltip"]["valueFormatter"] = "__BYTES_FORMATTER__" - elif panel_config.style.yUnit == "ms": - option["yAxis"]["axisLabel"]["formatter"] = "__MS_FORMATTER__" - option["tooltip"]["valueFormatter"] = "__MS_FORMATTER__" - elif panel_config.style.yUnit == "seconds": - option["yAxis"]["axisLabel"]["formatter"] = "__SECONDS_FORMATTER__" - option["tooltip"]["valueFormatter"] = "__SECONDS_FORMATTER__" - elif panel_config.style.yUnit == "percent": - option["yAxis"]["axisLabel"]["formatter"] = "__PERCENT_FORMATTER__" - option["tooltip"]["valueFormatter"] = "__PERCENT_FORMATTER__" - elif panel_config.style.yUnit == "number": - option["yAxis"]["axisLabel"]["formatter"] = "__NUMBER_FORMATTER__" - option["tooltip"]["valueFormatter"] = "__NUMBER_FORMATTER__" - - # Merge user-provided ECharts options (if any) - if panel_config.echarts_options: - option = deep_merge(option, panel_config.echarts_options) - - return option - - def transform_panel_data( - self, experiment_id: str, panel_config: PanelConfig, viz_format: str = "echarts" - ) -> Dict[str, Any]: - """Transform dataset for panel visualization into the requested viz_format.""" - dataset = self.db.get_dataset(experiment_id, panel_config.dataset) - if not dataset: - raise ValueError(f"Dataset '{panel_config.dataset}' not found") - - # Apply derive transformations - data = self._apply_derive_transformations(dataset, panel_config) - - # Route to appropriate transformer based on viz_format - if viz_format == "echarts": - return self._transform_to_echarts(data, panel_config) - elif viz_format == "plotly": - # Future: implement Plotly transformer - raise NotImplementedError("Plotly transformation not yet implemented") - else: - raise ValueError(f"Unsupported visualization format: {viz_format}") - - def _transform_to_echarts( - self, data: List[Dict[str, Any]], panel_config: PanelConfig - ) -> Dict[str, Any]: - """Transform data to ECharts format based on panel type.""" - if panel_config.type == "boxplot": - return self._transform_to_boxplot(data, panel_config) - elif panel_config.type == "timeseries": - return self._transform_to_timeseries(data, panel_config) - elif panel_config.type == "histogram": - # TODO: Implement histogram transformation - raise NotImplementedError("Histogram transformation not yet implemented") - elif panel_config.type == "bar": - # TODO: Implement bar chart transformation - raise NotImplementedError("Bar chart transformation not yet implemented") - elif panel_config.type == "table": - # For tables, just return the data as-is - return {"type": "table", "title": panel_config.title, "data": data} - else: - raise ValueError(f"Unknown panel type: {panel_config.type}") diff --git a/dst_dashboard/processors/tests/__init__.py b/dst_dashboard/processors/tests/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/dst_dashboard/processors/tests/test_panel_processor.py b/dst_dashboard/processors/tests/test_panel_processor.py deleted file mode 100644 index 641d00b3..00000000 --- a/dst_dashboard/processors/tests/test_panel_processor.py +++ /dev/null @@ -1,723 +0,0 @@ -from unittest.mock import MagicMock - -import pytest - -from dst_dashboard.config.data_structures import ( - DashboardFullConfig, - DeriveField, - ExperimentConfig, - PanelConfig, - PanelStyle, - PanelTransform, -) -from dst_dashboard.processors.panel_processor import PanelProcessor, deep_merge - -# --------------------------------------------------------------------------- # -# Helper Functions -# --------------------------------------------------------------------------- # - - -def _make_processor(db=None) -> PanelProcessor: - """Create a PanelProcessor with a minimal real config and a fake db.""" - config = DashboardFullConfig(datasources=[]) - return PanelProcessor(config, db if db is not None else MagicMock()) - - -def _create_panel_config( - name: str = "test-panel", - title: str = "Test Panel", - panel_type: str = "boxplot", - dataset: str = "test-dataset", - transform: PanelTransform = None, - style: PanelStyle = None, - echarts_options: dict = None, - publish: bool = True, -) -> PanelConfig: - """Create a real PanelConfig object for testing.""" - return PanelConfig( - name=name, - title=title, - type=panel_type, - dataset=dataset, - transform=transform if transform is not None else PanelTransform(), - style=style, - echarts_options=echarts_options, - publish=publish, - ) - - -def _create_experiment_config( - experiment_id: str = "exp-1", - panels: list = None, -) -> ExperimentConfig: - """Create a real ExperimentConfig object for testing.""" - return ExperimentConfig( - id=experiment_id, - title="Test Experiment", - family="test/family", - metadata={}, - datasets=[], - panels=panels or [], - publish=True, - ) - - -# --------------------------------------------------------------------------- # -# deep_merge Tests -# --------------------------------------------------------------------------- # -class TestDeepMerge: - """Tests for the deep_merge function.""" - - def test_merge_disjoint_keys_combines_both(self): - """Should include keys from both dictionaries when they don't overlap.""" - base = {"a": 1} - override = {"b": 2} - - assert deep_merge(base, override) == {"a": 1, "b": 2} - - def test_merge_overlapping_scalar_keys_prefers_override(self): - """Should let override values win for non-dict keys present in both.""" - base = {"a": 1, "b": 2} - override = {"b": 99} - - assert deep_merge(base, override) == {"a": 1, "b": 99} - - def test_merge_nested_dicts_recursively(self): - """Should recursively merge nested dictionaries instead of replacing them wholesale.""" - base = {"grid": {"left": "10%", "right": "10%"}} - override = {"grid": {"right": "5%"}} - - assert deep_merge(base, override) == {"grid": {"left": "10%", "right": "5%"}} - - def test_merge_does_not_mutate_inputs(self): - """Should leave the original base and override dictionaries untouched.""" - base = {"a": {"x": 1}} - override = {"a": {"y": 2}} - - deep_merge(base, override) - - assert base == {"a": {"x": 1}} - assert override == {"a": {"y": 2}} - - -# --------------------------------------------------------------------------- # -# PanelProcessor.process_panel Tests -# --------------------------------------------------------------------------- # -class TestProcessPanel: - """Tests for PanelProcessor.process_panel.""" - - def test_missing_dataset_returns_false_without_processing(self): - """Should skip processing and return False when the panel's dataset doesn't exist.""" - db = MagicMock() - db.dataset_exists.return_value = False - processor = _make_processor(db) - panel_config = _create_panel_config(dataset="missing-dataset") - - result = processor.process_panel("exp-1", panel_config) - - assert result is False - db.store_panel_data.assert_not_called() - - def test_successful_transform_stores_option_and_returns_true(self, mocker): - """Should store the transformed ECharts option and return True on success.""" - db = MagicMock() - db.dataset_exists.return_value = True - processor = _make_processor(db) - panel_config = _create_panel_config(name="my-panel") - echarts_option = {"title": {"text": "My Panel"}} - mocker.patch.object(processor, "transform_panel_data", return_value=echarts_option) - - result = processor.process_panel("exp-1", panel_config) - - assert result is True - db.store_panel_data.assert_called_once_with("exp-1", "my-panel", echarts_option) - - def test_transform_exception_is_caught_and_returns_false(self, mocker): - """Should catch exceptions from the transform step and return False instead of raising.""" - db = MagicMock() - db.dataset_exists.return_value = True - processor = _make_processor(db) - panel_config = _create_panel_config() - mocker.patch.object(processor, "transform_panel_data", side_effect=ValueError("boom")) - - result = processor.process_panel("exp-1", panel_config) - - assert result is False - db.store_panel_data.assert_not_called() - - -# --------------------------------------------------------------------------- # -# PanelProcessor.process_experiment_panels Tests -# --------------------------------------------------------------------------- # -class TestProcessExperimentPanels: - """Tests for PanelProcessor.process_experiment_panels.""" - - def test_no_panels_returns_zero(self): - """Should return 0 immediately when the experiment has no panels.""" - processor = _make_processor() - experiment = _create_experiment_config(panels=[]) - - assert processor.process_experiment_panels(experiment) == 0 - - def test_single_panel_processed_sequentially(self, mocker): - """Should process a lone panel directly without spinning up a thread pool.""" - processor = _make_processor() - panel = _create_panel_config(name="only-panel") - experiment = _create_experiment_config(panels=[panel]) - mock_process_panel = mocker.patch.object(processor, "process_panel", return_value=True) - - result = processor.process_experiment_panels(experiment) - - assert result == 1 - mock_process_panel.assert_called_once_with(experiment.id, panel) - - def test_max_workers_one_processes_sequentially(self, mocker): - """Should fall back to sequential processing when max_workers is 1.""" - processor = _make_processor() - panels = [_create_panel_config(name="p1"), _create_panel_config(name="p2")] - experiment = _create_experiment_config(panels=panels) - mocker.patch.object(processor, "process_panel", return_value=True) - - result = processor.process_experiment_panels(experiment, max_workers=1) - - assert result == 2 - - def test_multiple_panels_all_succeed_counts_all(self, mocker): - """Should process all panels concurrently and count every success.""" - processor = _make_processor() - panels = [_create_panel_config(name=f"p{i}") for i in range(4)] - experiment = _create_experiment_config(panels=panels) - mocker.patch.object(processor, "process_panel", return_value=True) - - result = processor.process_experiment_panels(experiment, max_workers=4) - - assert result == 4 - - def test_multiple_panels_mixed_results_counts_only_successes(self, mocker): - """Should only count panels that return True, not failed ones.""" - processor = _make_processor() - panels = [_create_panel_config(name=f"p{i}") for i in range(3)] - experiment = _create_experiment_config(panels=panels) - results_by_name = {"p0": True, "p1": False, "p2": True} - mocker.patch.object( - processor, - "process_panel", - side_effect=lambda experiment_id, panel_config: results_by_name[panel_config.name], - ) - - result = processor.process_experiment_panels(experiment, max_workers=4) - - assert result == 2 - - def test_panel_raising_unexpectedly_does_not_stop_others(self, mocker): - """Should skip a panel whose processing raises without losing the other results.""" - processor = _make_processor() - panels = [_create_panel_config(name="ok-panel"), _create_panel_config(name="bad-panel")] - experiment = _create_experiment_config(panels=panels) - - def fake_process_panel(experiment_id, panel_config): - if panel_config.name == "bad-panel": - raise RuntimeError("unexpected failure") - return True - - mocker.patch.object(processor, "process_panel", side_effect=fake_process_panel) - - result = processor.process_experiment_panels(experiment, max_workers=4) - - assert result == 1 - - -# --------------------------------------------------------------------------- # -# PanelProcessor._apply_derive_transformations Tests -# --------------------------------------------------------------------------- # -class TestApplyDeriveTransformations: - """Tests for PanelProcessor._apply_derive_transformations.""" - - def test_no_derive_rules_returns_data_unchanged(self): - """Should return the data unchanged when the panel's transform has no derive rules.""" - processor = _make_processor() - panel_config = _create_panel_config(transform=PanelTransform()) - data = [{"pod_name": "node-1"}] - - result = processor._apply_derive_transformations(data, panel_config) - - assert result == data - - def test_regex_match_assigns_match_value(self): - """Should assign the 'match' value when the regex pattern matches the field.""" - processor = _make_processor() - derive = DeriveField( - name="pod_group", - function="regex_match", - field="pod_name", - pattern=".*slow.*", - match="slow", - no_match="normal", - ) - panel_config = _create_panel_config(transform=PanelTransform(derive=[derive])) - data = [{"pod_name": "nimp2p-slow-1"}] - - result = processor._apply_derive_transformations(data, panel_config) - - assert result == [{"pod_name": "nimp2p-slow-1", "pod_group": "slow"}] - - def test_regex_no_match_assigns_no_match_value(self): - """Should assign the 'no_match' value when the regex pattern doesn't match the field.""" - processor = _make_processor() - derive = DeriveField( - name="pod_group", - function="regex_match", - field="pod_name", - pattern=".*slow.*", - match="slow", - no_match="normal", - ) - panel_config = _create_panel_config(transform=PanelTransform(derive=[derive])) - data = [{"pod_name": "nimp2p-fast-1"}] - - result = processor._apply_derive_transformations(data, panel_config) - - assert result == [{"pod_name": "nimp2p-fast-1", "pod_group": "normal"}] - - def test_unknown_derive_function_leaves_row_without_new_field(self): - """Should leave the row without the derived field when the derive function is unknown.""" - processor = _make_processor() - derive = DeriveField(name="foo", function="unsupported_function", field="pod_name") - panel_config = _create_panel_config(transform=PanelTransform(derive=[derive])) - data = [{"pod_name": "node-1"}] - - result = processor._apply_derive_transformations(data, panel_config) - - assert result == [{"pod_name": "node-1"}] - - def test_original_data_rows_are_not_mutated(self): - """Should return new row dicts instead of mutating the original input rows.""" - processor = _make_processor() - derive = DeriveField( - name="pod_group", - function="regex_match", - field="pod_name", - pattern=".*slow.*", - match="slow", - no_match="normal", - ) - panel_config = _create_panel_config(transform=PanelTransform(derive=[derive])) - original_row = {"pod_name": "nimp2p-slow-1"} - data = [original_row] - - processor._apply_derive_transformations(data, panel_config) - - assert original_row == {"pod_name": "nimp2p-slow-1"} - - -# --------------------------------------------------------------------------- # -# PanelProcessor._transform_to_boxplot Tests -# --------------------------------------------------------------------------- # -class TestTransformToBoxplot: - """Tests for PanelProcessor._transform_to_boxplot.""" - - def test_missing_groupby_raises_value_error(self): - """Should raise ValueError when 'groupBy' is missing from the transform.""" - processor = _make_processor() - panel_config = _create_panel_config(transform=PanelTransform(value="delayMs")) - - with pytest.raises(ValueError, match="requires 'groupBy' and 'value'"): - processor._transform_to_boxplot([], panel_config) - - def test_missing_value_raises_value_error(self): - """Should raise ValueError when 'value' is missing from the transform.""" - processor = _make_processor() - panel_config = _create_panel_config(transform=PanelTransform(groupBy="pod_name")) - - with pytest.raises(ValueError, match="requires 'groupBy' and 'value'"): - processor._transform_to_boxplot([], panel_config) - - def test_computes_boxplot_stats_per_group_sorted_by_category(self): - """Should compute [min, Q1, median, Q3, max] per group, sorted by category name.""" - processor = _make_processor() - panel_config = _create_panel_config( - transform=PanelTransform(groupBy="pod_name", value="delayMs") - ) - data = [ - {"pod_name": "b", "delayMs": 10}, - {"pod_name": "a", "delayMs": 1}, - {"pod_name": "a", "delayMs": 2}, - {"pod_name": "a", "delayMs": 3}, - ] - - option = processor._transform_to_boxplot(data, panel_config) - - assert option["xAxis"]["data"] == ["a", "b"] - series_data = option["series"][0]["data"] - assert series_data[0] == [1.0, 1.0, 2.0, 3.0, 3.0] - assert series_data[1] == [10.0, 10.0, 10.0, 10.0, 10.0] - - def test_top_filter_keeps_highest_average_groups(self): - """Should keep only the top-N groups by average value when 'top' is set.""" - processor = _make_processor() - panel_config = _create_panel_config( - transform=PanelTransform(groupBy="pod_name", value="delayMs", top=1) - ) - data = [ - {"pod_name": "low", "delayMs": 1}, - {"pod_name": "high", "delayMs": 100}, - ] - - option = processor._transform_to_boxplot(data, panel_config) - - assert option["xAxis"]["data"] == ["high"] - - def test_style_options_are_applied_to_axes(self): - """Should apply xLabel/yLabel/yMin/yMax from panel style onto the axes.""" - processor = _make_processor() - style = PanelStyle(xLabel="Pod", yLabel="Delay (ms)", yMin=0, yMax=100) - panel_config = _create_panel_config( - transform=PanelTransform(groupBy="pod_name", value="delayMs"), style=style - ) - data = [{"pod_name": "a", "delayMs": 1}] - - option = processor._transform_to_boxplot(data, panel_config) - - assert option["xAxis"]["name"] == "Pod" - assert option["yAxis"]["name"] == "Delay (ms)" - assert option["yAxis"]["min"] == 0 - assert option["yAxis"]["max"] == 100 - - def test_echarts_options_override_is_deep_merged(self): - """Should deep-merge user-provided echarts_options into the generated option.""" - processor = _make_processor() - panel_config = _create_panel_config( - transform=PanelTransform(groupBy="pod_name", value="delayMs"), - echarts_options={"grid": {"left": "5%"}}, - ) - data = [{"pod_name": "a", "delayMs": 1}] - - option = processor._transform_to_boxplot(data, panel_config) - - assert option["grid"]["left"] == "5%" - assert option["grid"]["right"] == "10%" # untouched default preserved - - def test_rows_missing_the_value_field_are_ignored(self): - """Should skip rows where the value field is missing/None instead of raising.""" - processor = _make_processor() - panel_config = _create_panel_config( - transform=PanelTransform(groupBy="pod_name", value="delayMs") - ) - data = [{"pod_name": "a", "delayMs": 5}, {"pod_name": "a"}] - - option = processor._transform_to_boxplot(data, panel_config) - - assert option["series"][0]["data"] == [[5.0, 5.0, 5.0, 5.0, 5.0]] - - -# --------------------------------------------------------------------------- # -# PanelProcessor._transform_to_timeseries Tests -# --------------------------------------------------------------------------- # -class TestTransformToTimeseries: - """Tests for PanelProcessor._transform_to_timeseries.""" - - def test_missing_x_raises_value_error(self): - """Should raise ValueError when 'x' is missing from the transform.""" - processor = _make_processor() - panel_config = _create_panel_config( - panel_type="timeseries", transform=PanelTransform(y="value") - ) - - with pytest.raises(ValueError, match="requires 'x' and 'y'"): - processor._transform_to_timeseries([], panel_config) - - def test_missing_y_raises_value_error(self): - """Should raise ValueError when 'y' is missing from the transform.""" - processor = _make_processor() - panel_config = _create_panel_config( - panel_type="timeseries", transform=PanelTransform(x="timestamp") - ) - - with pytest.raises(ValueError, match="requires 'x' and 'y'"): - processor._transform_to_timeseries([], panel_config) - - def test_grouped_series_are_sorted_by_name_with_cycling_colors(self): - """Should build one series per group, sorted alphabetically, cycling through the palette.""" - processor = _make_processor() - panel_config = _create_panel_config( - panel_type="timeseries", - transform=PanelTransform(x="ts", y="v", groupBy="pod_name"), - ) - data = [ - {"pod_name": "b", "ts": "t2", "v": 5}, - {"pod_name": "a", "ts": "t1", "v": 1}, - {"pod_name": "a", "ts": "t2", "v": 3}, - {"pod_name": "c", "ts": "t1", "v": 100}, - ] - - option = processor._transform_to_timeseries(data, panel_config) - - series = option["series"] - assert [s["name"] for s in series] == ["a", "b", "c"] - assert series[0]["data"] == [["t1", 1.0], ["t2", 3.0]] - assert series[1]["data"] == [["t2", 5.0]] - assert series[2]["data"] == [["t1", 100.0]] - assert series[0]["lineStyle"]["color"] == "#5470c6" - assert series[1]["lineStyle"]["color"] == "#91cc75" - assert series[2]["lineStyle"]["color"] == "#fac858" - - def test_top_filter_keeps_highest_average_series(self): - """Should keep only the top-N series by average value when 'top' is set.""" - processor = _make_processor() - panel_config = _create_panel_config( - panel_type="timeseries", - transform=PanelTransform(x="ts", y="v", groupBy="pod_name", top=2), - ) - data = [ - {"pod_name": "b", "ts": "t2", "v": 5}, - {"pod_name": "a", "ts": "t1", "v": 1}, - {"pod_name": "a", "ts": "t2", "v": 3}, - {"pod_name": "c", "ts": "t1", "v": 100}, - ] - - option = processor._transform_to_timeseries(data, panel_config) - - assert [s["name"] for s in option["series"]] == ["b", "c"] - - def test_firstn_filter_keeps_first_n_by_name(self): - """Should keep only the first-N series by name (no value sorting) when 'firstN' is set.""" - processor = _make_processor() - panel_config = _create_panel_config( - panel_type="timeseries", - transform=PanelTransform(x="ts", y="v", groupBy="pod_name", firstN=2), - ) - data = [ - {"pod_name": "b", "ts": "t2", "v": 5}, - {"pod_name": "a", "ts": "t1", "v": 1}, - {"pod_name": "c", "ts": "t1", "v": 100}, - ] - - option = processor._transform_to_timeseries(data, panel_config) - - assert [s["name"] for s in option["series"]] == ["a", "b"] - - def test_single_series_without_groupby_has_no_x_pairing(self): - """ - Characterizes current behavior: without groupBy, series data is a plain - list of y-values (no [timestamp, value] pairing), unlike the grouped path. - """ - processor = _make_processor() - panel_config = _create_panel_config( - panel_type="timeseries", title="Solo", transform=PanelTransform(x="ts", y="v") - ) - data = [{"ts": "t1", "v": 1}, {"ts": "t2", "v": 2}] - - option = processor._transform_to_timeseries(data, panel_config) - - assert option["series"] == [ - { - "name": "Solo", - "type": "line", - "data": [1.0, 2.0], - "smooth": True, - "sampling": "lttb", - "symbol": "none", - "lineStyle": {"width": 2, "color": "#00d4ff"}, - "areaStyle": {"color": "#00d4ff", "opacity": 0.15}, - } - ] - - def test_datetime_x_values_are_serialized_to_isoformat(self): - """Should convert datetime-like x values to ISO strings via .isoformat().""" - from datetime import datetime - - processor = _make_processor() - panel_config = _create_panel_config( - panel_type="timeseries", transform=PanelTransform(x="ts", y="v", groupBy="pod") - ) - data = [{"pod": "a", "ts": datetime(2026, 7, 5, 17, 47, 11), "v": 1}] - - option = processor._transform_to_timeseries(data, panel_config) - - assert option["series"][0]["data"] == [["2026-07-05T17:47:11", 1.0]] - - def test_yunit_style_sets_formatter_placeholders(self): - """Should set the matching formatter placeholder on yAxis and tooltip for known units.""" - processor = _make_processor() - style = PanelStyle(yUnit="ms") - panel_config = _create_panel_config( - panel_type="timeseries", - transform=PanelTransform(x="ts", y="v", groupBy="pod"), - style=style, - ) - data = [{"pod": "a", "ts": "t1", "v": 1}] - - option = processor._transform_to_timeseries(data, panel_config) - - assert option["yAxis"]["axisLabel"]["formatter"] == "__MS_FORMATTER__" - assert option["tooltip"]["valueFormatter"] == "__MS_FORMATTER__" - - def test_style_labels_are_applied_to_axes(self): - """Should apply xLabel/yLabel/yMin/yMax from panel style onto the axes.""" - processor = _make_processor() - style = PanelStyle(xLabel="Time", yLabel="Value", yMin=0, yMax=10) - panel_config = _create_panel_config( - panel_type="timeseries", - transform=PanelTransform(x="ts", y="v", groupBy="pod"), - style=style, - ) - data = [{"pod": "a", "ts": "t1", "v": 1}] - - option = processor._transform_to_timeseries(data, panel_config) - - assert option["xAxis"]["name"] == "Time" - assert option["yAxis"]["name"] == "Value" - assert option["yAxis"]["min"] == 0 - assert option["yAxis"]["max"] == 10 - - def test_echarts_options_override_is_deep_merged(self): - """Should deep-merge user-provided echarts_options into the generated option.""" - processor = _make_processor() - panel_config = _create_panel_config( - panel_type="timeseries", - transform=PanelTransform(x="ts", y="v", groupBy="pod"), - echarts_options={"grid": {"left": "1%"}}, - ) - data = [{"pod": "a", "ts": "t1", "v": 1}] - - option = processor._transform_to_timeseries(data, panel_config) - - assert option["grid"]["left"] == "1%" - assert option["grid"]["right"] == "4%" # untouched default preserved - - -# --------------------------------------------------------------------------- # -# PanelProcessor.transform_panel_data Tests -# --------------------------------------------------------------------------- # -class TestTransformPanelData: - """Tests for PanelProcessor.transform_panel_data.""" - - def test_missing_dataset_raises_value_error(self): - """Should raise ValueError when the panel's dataset has no stored data.""" - db = MagicMock() - db.get_dataset.return_value = None - processor = _make_processor(db) - panel_config = _create_panel_config(dataset="missing-dataset") - - with pytest.raises(ValueError, match="Dataset 'missing-dataset' not found"): - processor.transform_panel_data("exp-1", panel_config) - - def test_empty_dataset_raises_value_error(self): - """Should raise ValueError when the panel's dataset exists but has zero rows.""" - db = MagicMock() - db.get_dataset.return_value = [] - processor = _make_processor(db) - panel_config = _create_panel_config(dataset="empty-dataset") - - with pytest.raises(ValueError, match="Dataset 'empty-dataset' not found"): - processor.transform_panel_data("exp-1", panel_config) - - def test_applies_derive_transformations_before_routing_to_echarts(self, mocker): - """Should run derive transformations on the fetched data before building the option.""" - db = MagicMock() - db.get_dataset.return_value = [{"pod_name": "node-1"}] - processor = _make_processor(db) - panel_config = _create_panel_config() - derived_data = [{"pod_name": "node-1", "pod_group": "normal"}] - mock_derive = mocker.patch.object( - processor, "_apply_derive_transformations", return_value=derived_data - ) - mock_echarts = mocker.patch.object( - processor, "_transform_to_echarts", return_value={"ok": True} - ) - - result = processor.transform_panel_data("exp-1", panel_config) - - mock_derive.assert_called_once_with([{"pod_name": "node-1"}], panel_config) - mock_echarts.assert_called_once_with(derived_data, panel_config) - assert result == {"ok": True} - - def test_plotly_format_raises_not_implemented(self): - """Should raise NotImplementedError for the not-yet-supported plotly format.""" - db = MagicMock() - db.get_dataset.return_value = [{"a": 1}] - processor = _make_processor(db) - panel_config = _create_panel_config() - - with pytest.raises(NotImplementedError, match="Plotly"): - processor.transform_panel_data("exp-1", panel_config, viz_format="plotly") - - def test_unsupported_format_raises_value_error(self): - """Should raise ValueError for a viz_format that isn't recognized at all.""" - db = MagicMock() - db.get_dataset.return_value = [{"a": 1}] - processor = _make_processor(db) - panel_config = _create_panel_config() - - with pytest.raises(ValueError, match="Unsupported visualization format"): - processor.transform_panel_data("exp-1", panel_config, viz_format="svg") - - -# --------------------------------------------------------------------------- # -# PanelProcessor._transform_to_echarts Tests -# --------------------------------------------------------------------------- # -class TestTransformToEcharts: - """Tests for PanelProcessor._transform_to_echarts.""" - - def test_boxplot_type_routes_to_transform_to_boxplot(self, mocker): - """Should route boxplot panels to _transform_to_boxplot.""" - processor = _make_processor() - panel_config = _create_panel_config(panel_type="boxplot") - data = [{"a": 1}] - mock_boxplot = mocker.patch.object( - processor, "_transform_to_boxplot", return_value={"kind": "boxplot"} - ) - - result = processor._transform_to_echarts(data, panel_config) - - mock_boxplot.assert_called_once_with(data, panel_config) - assert result == {"kind": "boxplot"} - - def test_timeseries_type_routes_to_transform_to_timeseries(self, mocker): - """Should route timeseries panels to _transform_to_timeseries.""" - processor = _make_processor() - panel_config = _create_panel_config(panel_type="timeseries") - data = [{"a": 1}] - mock_timeseries = mocker.patch.object( - processor, "_transform_to_timeseries", return_value={"kind": "timeseries"} - ) - - result = processor._transform_to_echarts(data, panel_config) - - mock_timeseries.assert_called_once_with(data, panel_config) - assert result == {"kind": "timeseries"} - - def test_histogram_type_raises_not_implemented(self): - """Should raise NotImplementedError for the not-yet-supported histogram type.""" - processor = _make_processor() - panel_config = _create_panel_config(panel_type="histogram") - - with pytest.raises(NotImplementedError, match="Histogram"): - processor._transform_to_echarts([], panel_config) - - def test_bar_type_raises_not_implemented(self): - """Should raise NotImplementedError for the not-yet-supported bar type.""" - processor = _make_processor() - panel_config = _create_panel_config(panel_type="bar") - - with pytest.raises(NotImplementedError, match="Bar chart"): - processor._transform_to_echarts([], panel_config) - - def test_table_type_returns_data_as_is(self): - """Should return the data unchanged (wrapped with type/title) for table panels.""" - processor = _make_processor() - panel_config = _create_panel_config(panel_type="table", title="Raw Rows") - data = [{"a": 1}, {"a": 2}] - - result = processor._transform_to_echarts(data, panel_config) - - assert result == {"type": "table", "title": "Raw Rows", "data": data} - - def test_unknown_type_raises_value_error(self): - """Should raise ValueError for a panel type outside the known set.""" - processor = _make_processor() - panel_config = _create_panel_config(panel_type="boxplot") - panel_config.type = "unknown-type" # bypass pydantic's Literal at construction time - - with pytest.raises(ValueError, match="Unknown panel type"): - processor._transform_to_echarts([], panel_config) diff --git a/dst_dashboard/sample_config.yaml b/dst_dashboard/sample_config.yaml deleted file mode 100644 index ca5f860a..00000000 --- a/dst_dashboard/sample_config.yaml +++ /dev/null @@ -1,23 +0,0 @@ -# DST Explorer Configuration -# Global datasources - defined once, referenced by all experiments. -# -# Experiments are no longer defined here - they're managed exclusively through -# the API (see README.md "Managing experiments"). To seed the 3 example -# experiments this file used to contain, see -# dst_dashboard/scripts/seed_data/experiments.json and seed_experiments.py. - -datasources: - - name: victoria-logs - type: VictoriaLogs - url: https://vlselect.lab.vac.dev/select/logsql/query - - - name: victoria-metrics - type: Prometheus - url: https://metrics.lab.vac.dev/select/0/prometheus/api/v1/ - - - - - - - diff --git a/dst_dashboard/scripts/__init__.py b/dst_dashboard/scripts/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/dst_dashboard/scripts/seed_data/experiments.json b/dst_dashboard/scripts/seed_data/experiments.json deleted file mode 100644 index 6cc7977f..00000000 --- a/dst_dashboard/scripts/seed_data/experiments.json +++ /dev/null @@ -1,367 +0,0 @@ -{ - "experiments": [ - { - "title": "GossipSub Priority Queues: Medium Queue Overflow", - "family": "libp2p/gossipsub", - "publish": true, - "description": "Analysis of medium priority queue behavior under high load conditions with mixed node speeds", - "github_repo": "https://github.com/vacp2p/nim-libp2p", - "github_pr": "https://github.com/vacp2p/nim-libp2p/pull/1234", - "docker_image": "harbor.status.im/nim-libp2p:v1.2.0", - "date": "2026-06-12", - "metadata": { - "scenario": "S1", - "namespace": "libp2p-lab", - "date": "2026-06-12", - "num_normal_nodes": 1, - "num_slow_nodes": 1, - "statefulsets": [ - "nimp2p" - ], - "nodes": [ - 2 - ], - "container_name": "pod-0" - }, - "datasets": [ - { - "name": "message-delays", - "datasource": "victoria-logs", - "timeRange": { - "start": "2026-06-12T22:09:00Z", - "end": "2026-06-12T22:13:30Z" - }, - "query": { - "namespace": "libp2p-lab", - "tracer": "nimlibp2p", - "pattern": "received" - }, - "schema": [ - { - "name": "timestamp", - "type": "datetime" - }, - { - "name": "pod_name", - "type": "string" - }, - { - "name": "delayMs", - "type": "float" - } - ] - }, - { - "name": "queue-drops", - "datasource": "victoria-metrics", - "timeRange": { - "start": "2026-06-12T22:09:00Z", - "end": "2026-06-12T22:13:30Z" - }, - "query": { - "expr": "increase(libp2p_pubsub_medium_priority_queue_drops_total{namespace=\"libp2p-lab\"}[1m])", - "step": "15s", - "scrapeInterval": "10s" - }, - "schema": [ - { - "name": "timestamp", - "type": "datetime" - }, - { - "name": "pod", - "type": "string" - }, - { - "name": "value", - "type": "float" - } - ] - } - ], - "panels": [ - { - "name": "delay-boxplot", - "title": "Message Delivery Delay Distribution", - "type": "boxplot", - "dataset": "message-delays", - "publish": true, - "transform": { - "derive": [ - { - "name": "pod_group", - "function": "regex_match", - "field": "pod_name", - "pattern": ".*slow.*", - "match": "slow", - "no_match": "normal" - } - ], - "groupBy": "pod_group", - "value": "delayMs" - } - }, - { - "name": "medium-queue-drops", - "title": "Medium Queue Drops Over Time", - "type": "timeseries", - "dataset": "queue-drops", - "publish": true, - "transform": { - "x": "timestamp", - "y": "value", - "groupBy": "pod" - }, - "style": { - "yLabel": "Drops/sec" - } - } - ] - }, - { - "title": "Nimlibp2p Message Delivery Performance", - "family": "libp2p/nimlibp2p", - "publish": true, - "description": "Performance testing of message delivery latency in nimlibp2p under various network conditions", - "github_repo": "https://github.com/vacp2p/nim-libp2p", - "github_pr": "https://github.com/vacp2p/nim-libp2p/pull/1235", - "docker_image": "harbor.status.im/nim-libp2p:v1.2.1", - "date": "2026-06-15", - "metadata": { - "namespace": "nimlibp2p", - "num_nodes": 20, - "statefulsets": [ - "nodes" - ], - "nodes": [ - 20 - ] - }, - "datasets": [ - { - "name": "message-delays", - "datasource": "victoria-logs", - "timeRange": { - "start": "2026-06-13T00:09:00Z", - "end": "2026-06-13T00:13:00Z" - }, - "query": { - "namespace": "nimlibp2p", - "tracer": "nimlibp2p", - "pattern": "received" - }, - "schema": [ - { - "name": "timestamp", - "type": "datetime" - }, - { - "name": "pod_name", - "type": "string" - }, - { - "name": "delayMs", - "type": "float" - } - ] - } - ], - "panels": [ - { - "name": "delay-by-peer", - "title": "Message Delivery Delay by Peer (Top 10)", - "type": "boxplot", - "dataset": "message-delays", - "publish": true, - "transform": { - "groupBy": "pod_name", - "value": "delayMs", - "top": 10 - } - } - ] - }, - { - "title": "GossipSub 100 Node Regression Test", - "family": "libp2p/gossipsub", - "publish": true, - "description": "GossipSub performance with 100 nodes using yamux muxer", - "github_repo": "https://github.com/vacp2p/nim-libp2p", - "docker_image": "mamoutoudiarra/nim-libp2p-test:gossip-queues-v0.4", - "date": "2026-07-05", - "metadata": { - "scenario": "S0", - "namespace": "libp2p-lab", - "muxer": "yamux", - "num_normal_nodes": 100, - "num_slow_nodes": 0, - "total_peers": 10, - "connect_to": 6, - "gossipsub_d": 6, - "statefulsets": [ - "nimp2p" - ], - "nodes": [ - 100 - ], - "container_name": "pod-0" - }, - "datasets": [ - { - "name": "message-delays", - "datasource": "victoria-logs", - "timeRange": { - "start": "2026-07-05T17:47:11Z", - "end": "2026-07-05T17:55:00Z" - }, - "query": { - "namespace": "libp2p-lab", - "tracer": "nimlibp2p", - "pattern": "received" - }, - "schema": [ - { - "name": "timestamp", - "type": "datetime" - }, - { - "name": "pod_name", - "type": "string" - }, - { - "name": "delayMs", - "type": "float" - } - ] - }, - { - "name": "bandwidth-usage-dl", - "datasource": "victoria-metrics", - "timeRange": { - "start": "2026-07-05T17:47:11Z", - "end": "2026-07-05T17:55:00Z" - }, - "query": { - "expr": "rate(container_network_transmit_bytes_total{namespace=\"libp2p-lab\"}[1m])", - "step": "15s", - "scrapeInterval": "10s" - }, - "schema": [ - { - "name": "timestamp", - "type": "datetime" - }, - { - "name": "pod", - "type": "string" - }, - { - "name": "value", - "type": "float" - } - ] - }, - { - "name": "bandwidth-usage-ul", - "datasource": "victoria-metrics", - "timeRange": { - "start": "2026-07-05T17:47:11Z", - "end": "2026-07-05T17:55:00Z" - }, - "query": { - "expr": "rate(container_network_receive_bytes_total{namespace=\"libp2p-lab\"}[1m])", - "step": "15s", - "scrapeInterval": "10s" - }, - "schema": [ - { - "name": "timestamp", - "type": "datetime" - }, - { - "name": "pod", - "type": "string" - }, - { - "name": "value", - "type": "float" - } - ] - } - ], - "panels": [ - { - "name": "delay-boxplot-top10", - "title": "Message Delivery Delay (Top 10 Peers)", - "type": "boxplot", - "dataset": "message-delays", - "publish": true, - "transform": { - "groupBy": "pod_name", - "value": "delayMs", - "top": 10 - } - }, - { - "name": "delay-timeseries-top10", - "title": "Message Delay Over Time (Top 10 Peers)", - "type": "timeseries", - "dataset": "message-delays", - "publish": true, - "transform": { - "x": "timestamp", - "y": "delayMs", - "groupBy": "pod_name", - "top": 10 - }, - "style": { - "yLabel": "Delay (ms)" - }, - "echarts_options": { - "color": [ - "#5470c6", - "#91cc75", - "#fac858", - "#ee6666", - "#73c0de" - ] - } - }, - { - "name": "bandwidth-usage-dl-chart", - "title": "Network Bandwidth Usage DL", - "type": "timeseries", - "dataset": "bandwidth-usage-dl", - "publish": true, - "transform": { - "x": "timestamp", - "y": "value", - "groupBy": "pod", - "top": 10 - }, - "style": { - "yLabel": "Download", - "yUnit": "bytes/s" - } - }, - { - "name": "bandwidth-usage-ul-chart", - "title": "Network Bandwidth Usage UL", - "type": "timeseries", - "dataset": "bandwidth-usage-ul", - "publish": true, - "transform": { - "x": "timestamp", - "y": "value", - "groupBy": "pod", - "top": 10 - }, - "style": { - "yLabel": "Upload", - "yUnit": "bytes/s" - } - } - ] - } - ] -} \ No newline at end of file diff --git a/dst_dashboard/scripts/seed_experiments.py b/dst_dashboard/scripts/seed_experiments.py deleted file mode 100644 index ccd2a32a..00000000 --- a/dst_dashboard/scripts/seed_experiments.py +++ /dev/null @@ -1,94 +0,0 @@ -"""One-time (or re-run anytime) seeding of experiments via the API. - -Experiments are managed exclusively through the API now - config.yaml no -longer defines them. This script reads a JSON file of experiment definitions -(see seed_data/experiments.json, extracted from the old config.yaml) and -POSTs each one through POST /experiments, same as any other client would. -`id` is server-assigned on create, so the seed file doesn't (and shouldn't) -include one; a 409 means an experiment with that title already exists. - -Usage: - DST_ADMIN_TOKEN= uv run python -m dst_dashboard.scripts.seed_experiments - - or: - - uv run python -m dst_dashboard.scripts.seed_experiments \\ - --api-url http://localhost:8000 \\ - --token \\ - --seed-file dst_dashboard/scripts/seed_data/experiments.json -""" - -import argparse -import json -import os -import sys -from pathlib import Path - -import requests - -DEFAULT_SEED_FILE = Path(__file__).parent / "seed_data" / "experiments.json" - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "--api-url", - default=os.environ.get("DST_API_URL", "http://localhost:8000"), - help="Base URL of the running DST Dashboard API (default: %(default)s, or DST_API_URL)", - ) - parser.add_argument( - "--token", - default=os.environ.get("DST_ADMIN_TOKEN"), - help="Admin bearer token from /admin/token (default: DST_ADMIN_TOKEN env var)", - ) - parser.add_argument( - "--seed-file", - default=DEFAULT_SEED_FILE, - type=Path, - help="JSON file with an 'experiments:' list (default: %(default)s)", - ) - args = parser.parse_args() - - if not args.token: - print( - "Error: an admin token is required (--token or DST_ADMIN_TOKEN). " - "Get one from GET /admin/token.", - file=sys.stderr, - ) - sys.exit(1) - - with open(args.seed_file, "r", encoding="utf-8") as f: - seed_data = json.load(f) - - experiments = seed_data.get("experiments", []) - if not experiments: - print(f"No experiments found in {args.seed_file}") - return - - headers = {"Authorization": f"Bearer {args.token}"} - created, skipped, failed = 0, 0, 0 - - for experiment in experiments: - title = experiment.get("title", "") - response = requests.post( - f"{args.api_url}/experiments", json=experiment, headers=headers, timeout=120 - ) - - if response.status_code == 201: - new_id = response.json().get("id") - print(f"Created '{title}' (id={new_id})") - created += 1 - elif response.status_code == 409: - print(f"Skipped '{title}' (title already exists)") - skipped += 1 - else: - print(f"Failed '{title}': {response.status_code} {response.text}", file=sys.stderr) - failed += 1 - - print(f"\nDone: {created} created, {skipped} skipped, {failed} failed") - if failed: - sys.exit(1) - - -if __name__ == "__main__": - main() diff --git a/dst_dashboard/static/admin.html b/dst_dashboard/static/admin.html deleted file mode 100644 index aa833af9..00000000 --- a/dst_dashboard/static/admin.html +++ /dev/null @@ -1,384 +0,0 @@ - - - - - -DST Dashboard - Admin - - - - - - - -
-
- VacLab - DST Dashboard -
- VacLab · IFT -

Admin

- -
-

API Token

-

- Tokens are valid for 24 hours. Generate one below - it's used automatically - by the experiment form below for the rest of this page's session. -

-
No token yet - click "Generate token".
- - -

- Use elsewhere as: Authorization: Bearer <token> -

-
- -
-

Experiments

-

List, create, edit, or delete experiments by pasting JSON.

- - - - - - - - - - - - -
TitleFamilyIDDate
- -
-

New experiment

- -
- - -
-
-
- - - - diff --git a/dst_dashboard/storage/__init__.py b/dst_dashboard/storage/__init__.py deleted file mode 100644 index 8b137891..00000000 --- a/dst_dashboard/storage/__init__.py +++ /dev/null @@ -1 +0,0 @@ - diff --git a/dst_dashboard/storage/db.py b/dst_dashboard/storage/db.py deleted file mode 100644 index bd471dfc..00000000 --- a/dst_dashboard/storage/db.py +++ /dev/null @@ -1,232 +0,0 @@ -"""MongoDB database client for storing experiments and datasets.""" - -import json -import threading -from typing import Any, Dict, List, Optional - -from gridfs import GridFSBucket, NoFile -from pymongo import MongoClient - -from dst_dashboard.config.constants import Constants -from dst_dashboard.config.data_structures import DataSourceConfig - -# Process-wide MongoClient. MongoClient manages its own internal connection -# pool and is thread-safe, so a single instance is created lazily on first use -# and reused by every DSTDatabase() instantiation for the life of the process -# instead of opening a new connection per call. -_client_lock = threading.Lock() -_client: Optional[MongoClient] = None -_indexes_ensured = False - - -def _get_client() -> MongoClient: - """Get (creating if needed) the shared, process-wide MongoClient.""" - global _client - if _client is None: - with _client_lock: - if _client is None: - _client = MongoClient(str(Constants.DST_MONGO_URI)) - return _client - - -def _json_default(value: Any) -> str: - """json.dumps fallback for datetime-like objects (e.g. pandas Timestamp). - - BSON serializes these natively via isinstance(value, datetime.datetime), - but plain json.dumps doesn't - GridFS stores raw JSON bytes, so this - fills that gap the same way FastAPI's own encoder would. - """ - if hasattr(value, "isoformat"): - return value.isoformat() - raise TypeError(f"Object of type {type(value).__name__} is not JSON serializable") - - -class DSTDatabase: - """Database client for DST Dashboard, backed by MongoDB.""" - - def __init__(self): - """Initialize database client using the shared MongoClient.""" - self.client = _get_client() - self.db = self.client[str(Constants.DST_MONGO_DB_NAME)] - - # Collections - self.experiments = self.db.experiments - self.datasources = self.db.datasources - - # Dataset rows and panel ECharts specs can exceed MongoDB's 16MB - # document limit (millions of rows, or huge "top N" series arrays), - # so both are stored via GridFS, which transparently chunks large - # content and reassembles it on read. - self.dataset_fs = GridFSBucket(self.db, bucket_name="datasets") - self.panel_fs = GridFSBucket(self.db, bucket_name="panels") - - self._ensure_indexes() - - def _ensure_indexes(self): - """Create unique indexes once per process. Cheap no-op on subsequent calls.""" - global _indexes_ensured - if _indexes_ensured: - return - with _client_lock: - if _indexes_ensured: - return - self.experiments.create_index("id", unique=True) - self.experiments.create_index("title", unique=True) - self.datasources.create_index("name", unique=True) - self.db["datasets.files"].create_index("metadata.experiment_id") - self.db["panels.files"].create_index("metadata.experiment_id") - _indexes_ensured = True - - def store_experiment(self, experiment: Dict[str, Any]) -> str: - """Store experiment configuration. Returns the experiment ID.""" - self.experiments.update_one({"id": experiment["id"]}, {"$set": experiment}, upsert=True) - return experiment["id"] - - def get_experiment(self, experiment_id: str) -> Optional[Dict[str, Any]]: - """Get experiment by ID.""" - return self.experiments.find_one({"id": experiment_id}, {"_id": 0}) - - def list_experiments(self) -> List[Dict[str, Any]]: - """List all experiments.""" - return list(self.experiments.find({}, {"_id": 0})) - - def store_dataset( - self, experiment_id: str, dataset_name: str, data: List[Dict[str, Any]] - ) -> str: - """Store dataset data via GridFS. Returns the dataset ID.""" - dataset_id = f"{experiment_id}:{dataset_name}" - self._delete_gridfs_file(self.dataset_fs, dataset_id) - self.dataset_fs.upload_from_stream( - dataset_id, - json.dumps(data, default=_json_default).encode("utf-8"), - metadata={ - "experiment_id": experiment_id, - "name": dataset_name, - "row_count": len(data), - }, - ) - return dataset_id - - def get_dataset(self, experiment_id: str, dataset_name: str) -> Optional[List[Dict[str, Any]]]: - """Get dataset data.""" - dataset_id = f"{experiment_id}:{dataset_name}" - payload = self._download_gridfs_file(self.dataset_fs, dataset_id) - return json.loads(payload) if payload is not None else None - - def list_panels(self, experiment_id: str) -> List[Dict[str, Any]]: - """List stored panel metadata for an experiment.""" - docs = self.db["panels.files"].find( - {"metadata.experiment_id": experiment_id}, {"_id": 0, "filename": 1, "metadata": 1} - ) - return [ - { - "id": doc["filename"], - "experiment_id": doc["metadata"]["experiment_id"], - "name": doc["metadata"]["name"], - } - for doc in docs - ] - - def store_panel_data(self, experiment_id: str, panel_name: str, data: Dict[str, Any]) -> str: - """Store transformed panel data via GridFS. Returns the panel data ID.""" - panel_id = f"{experiment_id}:{panel_name}" - self._delete_gridfs_file(self.panel_fs, panel_id) - self.panel_fs.upload_from_stream( - panel_id, - json.dumps(data, default=_json_default).encode("utf-8"), - metadata={"experiment_id": experiment_id, "name": panel_name}, - ) - return panel_id - - def get_panel_data(self, experiment_id: str, panel_name: str) -> Optional[Dict[str, Any]]: - """Get transformed panel data.""" - panel_id = f"{experiment_id}:{panel_name}" - payload = self._download_gridfs_file(self.panel_fs, panel_id) - return json.loads(payload) if payload is not None else None - - def insert_datasource(self, datasource: DataSourceConfig) -> DataSourceConfig: - """Store datasource configuration.""" - data = datasource.model_dump() - self.datasources.update_one({"name": data["name"]}, {"$set": data}, upsert=True) - return datasource - - def insert_datasource_list(self, datasources: List[DataSourceConfig]) -> List[DataSourceConfig]: - """Store a list of datasource configurations.""" - for datasource in datasources: - self.insert_datasource(datasource) - return datasources - - def dataset_exists(self, experiment_id: str, dataset_name: str) -> bool: - """Check if dataset exists in database.""" - dataset_id = f"{experiment_id}:{dataset_name}" - return self.db["datasets.files"].find_one({"filename": dataset_id}, {"_id": 1}) is not None - - def get_dataset_metadata( - self, experiment_id: str, dataset_name: str - ) -> Optional[Dict[str, Any]]: - """Get dataset metadata without data rows.""" - dataset_id = f"{experiment_id}:{dataset_name}" - doc = self.db["datasets.files"].find_one( - {"filename": dataset_id}, {"_id": 0, "filename": 1, "metadata": 1} - ) - if doc is None: - return None - return { - "id": doc["filename"], - "experiment_id": doc["metadata"]["experiment_id"], - "name": doc["metadata"]["name"], - "row_count": doc["metadata"]["row_count"], - } - - def delete_experiment(self, experiment_id: str) -> bool: - """Delete an experiment and cascade to its datasets and panels.""" - result = self.experiments.delete_one({"id": experiment_id}) - self._delete_gridfs_files_matching( - self.dataset_fs, "datasets", {"metadata.experiment_id": experiment_id} - ) - self._delete_gridfs_files_matching( - self.panel_fs, "panels", {"metadata.experiment_id": experiment_id} - ) - return result.deleted_count > 0 - - def delete_dataset(self, experiment_id: str, dataset_name: str) -> bool: - """Delete a dataset.""" - dataset_id = f"{experiment_id}:{dataset_name}" - return self._delete_gridfs_file(self.dataset_fs, dataset_id) - - def delete_panel(self, experiment_id: str, panel_name: str) -> bool: - """Delete a panel.""" - panel_id = f"{experiment_id}:{panel_name}" - return self._delete_gridfs_file(self.panel_fs, panel_id) - - def clear_all_dataset_cache(self) -> int: - """Delete all cached dataset data across every experiment. Returns count removed.""" - count = self.db["datasets.files"].count_documents({}) - self.db["datasets.chunks"].delete_many({}) - self.db["datasets.files"].delete_many({}) - return count - - @staticmethod - def _delete_gridfs_file(bucket: GridFSBucket, filename: str) -> bool: - """Delete a GridFS file by filename. Returns True if a file was found and deleted.""" - deleted = False - for file_doc in bucket.find({"filename": filename}): - bucket.delete(file_doc._id) - deleted = True - return deleted - - def _delete_gridfs_files_matching( - self, bucket: GridFSBucket, bucket_name: str, query: Dict[str, Any] - ) -> None: - """Delete every GridFS file in a bucket matching a metadata query.""" - file_ids = [doc["_id"] for doc in self.db[f"{bucket_name}.files"].find(query, {"_id": 1})] - for file_id in file_ids: - bucket.delete(file_id) - - @staticmethod - def _download_gridfs_file(bucket: GridFSBucket, filename: str) -> Optional[bytes]: - """Download a GridFS file's raw bytes by filename, or None if it doesn't exist.""" - try: - return bucket.open_download_stream_by_name(filename).read() - except NoFile: - return None From 9db16d9c0ea5aa9f74563b490dd05370ba1ee200 Mon Sep 17 00:00:00 2001 From: Mamoutou Diarra Date: Tue, 14 Jul 2026 17:17:27 +0200 Subject: [PATCH 2/2] remove unused pages --- .../src/pages/ExperimentPage_backup.js | 231 ----------- .../frontend/src/pages/HomePage.js.backup | 380 ------------------ 2 files changed, 611 deletions(-) delete mode 100644 dst_dashboard/frontend/src/pages/ExperimentPage_backup.js delete mode 100644 dst_dashboard/frontend/src/pages/HomePage.js.backup diff --git a/dst_dashboard/frontend/src/pages/ExperimentPage_backup.js b/dst_dashboard/frontend/src/pages/ExperimentPage_backup.js deleted file mode 100644 index 5312c614..00000000 --- a/dst_dashboard/frontend/src/pages/ExperimentPage_backup.js +++ /dev/null @@ -1,231 +0,0 @@ -import React, { useState, useEffect } from 'react'; -import { useParams } from 'react-router-dom'; -import axios from 'axios'; -import ReactECharts from 'echarts-for-react'; -import Sidebar from '../components/Sidebar'; -import { API_BASE_URL } from '../config'; - -function ExperimentPage() { - const { experimentId } = useParams(); - const [experiment, setExperiment] = useState(null); - const [panels, setPanels] = useState([]); - const [loading, setLoading] = useState(true); - const [error, setError] = useState(null); - const [sidebarCollapsed, setSidebarCollapsed] = useState(false); - - useEffect(() => { - const fetchData = async() => { - try { - const expResponse = await axios.get(`${API_BASE_URL}/experiments/${experimentId}`); - setExperiment(expResponse.data); - - const panelsResponse = await axios.get(`${API_BASE_URL}/experiments/${experimentId}/panels`); - setPanels(panelsResponse.data.panels || []); - setLoading(false); - } catch (err) { - setError(err.message); - setLoading(false); - } - }; - fetchData(); - }, [experimentId]); - - if (loading) { - return ( < - div className = "App" > - < - Sidebar isCollapsed = { sidebarCollapsed } - onToggle = { - () => setSidebarCollapsed(!sidebarCollapsed) } - /> < - div className = { `main-content ${sidebarCollapsed ? 'sidebar-collapsed' : ''}` } > - < - div className = "container" > - < - div className = "loading" > Loading experiment... < /div> < - /div> < - /div> < - /div> - ); - } - - if (error) { - return ( < - div className = "App" > - < - Sidebar isCollapsed = { sidebarCollapsed } - onToggle = { - () => setSidebarCollapsed(!sidebarCollapsed) } - /> < - div className = { `main-content ${sidebarCollapsed ? 'sidebar-collapsed' : ''}` } > - < - div className = "container" > - < - div className = "error" > Error: { error } < /div> < - /div> < - /div> < - /div> - ); - } - - return ( < - div className = "App" > - < - Sidebar isCollapsed = { sidebarCollapsed } - onToggle = { - () => setSidebarCollapsed(!sidebarCollapsed) } - /> < - div className = { `main-content ${sidebarCollapsed ? 'sidebar-collapsed' : ''}` } > - < - div className = "page-header" > - < - div className = "header-content" > - < - div className = "experiment-category-label" > { experiment.family } < /div> < - h1 className = "page-title" > { experiment.title } < /h1> { - experiment.description && ( < - p className = "page-subtitle" > { experiment.description } < /p> - ) - } < - div className = "experiment-meta-links" > { - experiment.github_repo && ( < - a href = { experiment.github_repo } - className = "meta-link" - target = "_blank" - rel = "noopener noreferrer" > - < - svg width = "14" - height = "14" - viewBox = "0 0 16 16" - fill = "currentColor" > - < - path d = "M8 0C3.58 0 0 3.58 0 8c0 3.54 2.29 6.53 5.47 7.59.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2.01.37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.59.82-2.15-.08-.2-.36-1.02.08-2.12 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.92.08 2.12.51.56.82 1.27.82 2.15 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.46.55.38A8.013 8.013 0 0016 8c0-4.42-3.58-8-8-8z" / > - < - /svg> - GitHub < - /a> - ) - } { - experiment.github_pr && ( < - a href = { experiment.github_pr } - className = "meta-link" - target = "_blank" - rel = "noopener noreferrer" > - 🔀PR < - /a> - ) - } { - experiment.docker_image && ( < - span className = "meta-badge" > 🐳{ experiment.docker_image } < /span> - ) - } { - experiment.date && ( < - span className = "meta-badge" > 📅{ new Date(experiment.date).toLocaleDateString() } < /span> - ) - } < - /div> < - /div> < - /div> - - < - div className = "container" > - < - div className = "panels-section" > - < -
- {panels.map((panel) => { - // Process panel option to inject JavaScript formatters - const processedOption = JSON.parse(JSON.stringify(panel.option)); - - // Define formatter functions - const bytesFormatter = (value) => { - if (value >= 1073741824) return (value / 1073741824).toFixed(2) + ' GB/s'; - if (value >= 1048576) return (value / 1048576).toFixed(2) + ' MB/s'; - if (value >= 1024) return (value / 1024).toFixed(2) + ' KB/s'; - return value.toFixed(2) + ' B/s'; - }; - - const msFormatter = (value) => { - if (value >= 1000) return (value / 1000).toFixed(2) + ' s'; - return value.toFixed(2) + ' ms'; - }; - - const secondsFormatter = (value) => { - if (value >= 3600) return (value / 3600).toFixed(2) + ' h'; - if (value >= 60) return (value / 60).toFixed(2) + ' m'; - return value.toFixed(2) + ' s'; - }; - - const percentFormatter = (value) => { - return value.toFixed(2) + '%'; - }; - - const numberFormatter = (value) => { - if (value >= 1000000000) return (value / 1000000000).toFixed(2) + 'B'; - if (value >= 1000000) return (value / 1000000).toFixed(2) + 'M'; - if (value >= 1000) return (value / 1000).toFixed(2) + 'K'; - return value.toFixed(0); - }; - - // Replace formatter markers with actual functions - if (processedOption.yAxis?.axisLabel?.formatter === '__BYTES_FORMATTER__') { - processedOption.yAxis.axisLabel.formatter = bytesFormatter; - } - if (processedOption.tooltip?.valueFormatter === '__BYTES_FORMATTER__') { - processedOption.tooltip.valueFormatter = bytesFormatter; - } - - if (processedOption.yAxis?.axisLabel?.formatter === '__MS_FORMATTER__') { - processedOption.yAxis.axisLabel.formatter = msFormatter; - } - if (processedOption.tooltip?.valueFormatter === '__MS_FORMATTER__') { - processedOption.tooltip.valueFormatter = msFormatter; - } - - if (processedOption.yAxis?.axisLabel?.formatter === '__SECONDS_FORMATTER__') { - processedOption.yAxis.axisLabel.formatter = secondsFormatter; - } - if (processedOption.tooltip?.valueFormatter === '__SECONDS_FORMATTER__') { - processedOption.tooltip.valueFormatter = secondsFormatter; - } - - if (processedOption.yAxis?.axisLabel?.formatter === '__PERCENT_FORMATTER__') { - processedOption.yAxis.axisLabel.formatter = percentFormatter; - } - if (processedOption.tooltip?.valueFormatter === '__PERCENT_FORMATTER__') { - processedOption.tooltip.valueFormatter = percentFormatter; - } - - if (processedOption.yAxis?.axisLabel?.formatter === '__NUMBER_FORMATTER__') { - processedOption.yAxis.axisLabel.formatter = numberFormatter; - } - if (processedOption.tooltip?.valueFormatter === '__NUMBER_FORMATTER__') { - processedOption.tooltip.valueFormatter = numberFormatter; - } - - return ( -
-

{panel.panel_title}

- {panel.error ? ( -
Error: {panel.error}
- ) : ( - - )} -
- ); - })} -
- - - - - ); -} - -export default ExperimentPage; \ No newline at end of file diff --git a/dst_dashboard/frontend/src/pages/HomePage.js.backup b/dst_dashboard/frontend/src/pages/HomePage.js.backup deleted file mode 100644 index 3c86893f..00000000 --- a/dst_dashboard/frontend/src/pages/HomePage.js.backup +++ /dev/null @@ -1,380 +0,0 @@ -import React, { useState, useEffect } from 'react'; -import { useNavigate } from 'react-router-dom'; -import axios from 'axios'; -import ReactECharts from 'echarts-for-react'; -import Sidebar from '../components/Sidebar'; - -function HomePage() { - const [familiesData, setFamiliesData] = useState(null); - const [loading, setLoading] = useState(true); - const [error, setError] = useState(null); - const [selectedCategory, setSelectedCategory] = useState('all'); - const [sidebarCollapsed, setSidebarCollapsed] = useState(false); - const navigate = useNavigate(); - - useEffect(() => { - fetchFamilies(); - }, []); - - const fetchFamilies = async() => { - try { - const response = await axios.get('/experiments/families'); - setFamiliesData(response.data); - setLoading(false); - } catch (err) { - setError(err.message); - setLoading(false); - } - }; - - if (loading) { - return ( < - div className = "App" > - < - Sidebar isCollapsed = { sidebarCollapsed } - onToggle = { - () => setSidebarCollapsed(!sidebarCollapsed) - } - /> < - div className = { `main-content ${sidebarCollapsed ? 'sidebar-collapsed' : ''}` } > - < - div className = "container" > - < - div className = "loading" > Loading experiments... < /div> < / - div > < - /div> < / - div > - ); - } - - if (error) { - return ( < - div className = "App" > - < - Sidebar isCollapsed = { sidebarCollapsed } - onToggle = { - () => setSidebarCollapsed(!sidebarCollapsed) - } - /> < - div className = { `main-content ${sidebarCollapsed ? 'sidebar-collapsed' : ''}` } > - < - div className = "container" > - < - div className = "error" > Error loading experiments: { error } < /div> < / - div > < - /div> < / - div > - ); - } - - const totalExperiments = familiesData ? .total_experiments || 0; - const totalFamilies = familiesData ? .total_families || 0; - - const allExperiments = []; - familiesData ? .families ? .forEach((family) => { - family.experiments ? .forEach((exp) => { - allExperiments.push({ - ...exp, - family: family.name - }); - }); - }); - - const filteredExperiments = selectedCategory === 'all' ? - allExperiments : - allExperiments.filter(exp => exp.family === selectedCategory); - - const getTimelineOption = () => { - const sorted = [...allExperiments] - .filter(exp => exp.date) - .sort((a, b) => new Date(a.date) - new Date(b.date)); - - if (sorted.length === 0) { - return { - title: { text: 'No timeline data', left: 'center', top: 'center', textStyle: { color: '#666666', fontSize: 14 } } - }; - } - - const timelineData = sorted.map((exp, idx) => ({ - value: [exp.date, 0], - title: exp.title, - family: exp.family, - date: exp.date, - itemStyle: { - color: idx % 2 === 0 ? '#ffffff' : '#999999', - borderColor: '#000000', - borderWidth: 3, - shadowBlur: 15, - shadowColor: 'rgba(255, 255, 255, 0.4)' - } - })); - - return { - backgroundColor: 'transparent', - grid: { - left: '10%', - right: '10%', - top: '20%', - bottom: '20%', - containLabel: true - }, - xAxis: { - type: 'time', - boundaryGap: false, - axisLine: { - lineStyle: { - color: '#666666', - width: 1 - } - }, - axisTick: { - show: false - }, - axisLabel: { - color: '#999999', - fontSize: 11, - formatter: (value) => { - const date = new Date(value); - return date.toLocaleDateString('en-US', { - month: 'short', - day: 'numeric' - }); - } - }, - splitLine: { - show: false - } - }, - yAxis: { - show: false, - type: 'value' - }, - tooltip: { - trigger: 'item', - backgroundColor: '#1a1a1a', - borderColor: '#333333', - borderWidth: 1, - textStyle: { - color: '#ffffff', - fontSize: 12 - }, - formatter: (params) => { - return ` -
-
${params.data.title}
-
${params.data.family}
-
${new Date(params.data.date).toLocaleDateString('en-US', { - month: 'short', - day: 'numeric' - })}
-
- `; - } - }, - series: [{ - type: 'line', - data: timelineData, - lineStyle: { - color: '#666666', - width: 2 - }, - symbol: 'none', - smooth: false, - z: 1 - }, - { - type: 'scatter', - data: timelineData, - symbolSize: 10, - label: { - show: false - }, - emphasis: { - scale: 1.8, - itemStyle: { - shadowBlur: 15, - shadowColor: 'rgba(255, 255, 255, 0.8)', - borderWidth: 4 - } - }, - z: 2 - } - ] - }; - }; - - return ( < - div className = "App" > - < - Sidebar isCollapsed = { sidebarCollapsed } - onToggle = { - () => setSidebarCollapsed(!sidebarCollapsed) - } - /> < - div className = { `main-content ${sidebarCollapsed ? 'sidebar-collapsed' : ''}` } > - < - div className = "page-header" > - < - div className = "header-content" > - < - h1 className = "page-title" > DST Dashboard < /h1> < - p className = "page-subtitle" > Distributed Systems Testing Analytics < /p> < / - div > < - /div> - - < - div className = "overview-section" > - < - div className = "hero-stats" > - < - div className = "hero-stat-card" > - < - p className = "hero-stat-value" > { totalExperiments } < /p> < - p className = "hero-stat-label" > Total Experiments < /p> < - p className = "hero-stat-date" > Last updated { new Date().toLocaleDateString() } < /p> < / - div > < - div className = "hero-stat-card" > - < - p className = "hero-stat-value" > { totalFamilies } < /p> < - p className = "hero-stat-label" > Categories < /p> < - p className = "hero-stat-date" > Last updated { new Date().toLocaleDateString() } < /p> < / - div > < - /div> - - { - /* allExperiments.some(exp => exp.date) && ( < - div className = "timeline-section" > - < - h3 className = "timeline-title" > Timeline < /h3> < - div className = "chart-container" > - < - ReactECharts option = { getTimelineOption() } - style = { - { height: '200px', width: '100%' } } - notMerge = { true } - lazyUpdate = { true } - /> < - /div> < - /div> - ) */ - } < - /div> - - < - div className = "container" > - < - div className = "categories-section" > - < - h2 className = "section-title" > Categories < /h2> < - div className = "category-boxes" > - < - div className = { `category-box ${selectedCategory === 'all' ? 'active' : ''}` } - onClick = { - () => setSelectedCategory('all') - } > - < - span className = "category-box-name" > All Experiments < /span> < - span className = "category-box-count" > { allExperiments.length } < /span> < / - div > { - familiesData ? .families ? .map((family) => ( < - div key = { family.name } - className = { `category-box ${selectedCategory === family.name ? 'active' : ''}` } - onClick = { - () => setSelectedCategory(family.name) - } > - < - span className = "category-box-name" > { family.name } < /span> < - span className = "category-box-count" > { family.experiments ? .length || 0 } < /span> < / - div > - )) - } < - /div> < / - div > - - < - div className = "experiments-section" > - < - h2 className = "section-title" > { selectedCategory === 'all' ? 'All Experiments' : `${selectedCategory} Experiments` } < - /h2> < - div className = "experiments-grid" > { - filteredExperiments.map((experiment) => ( < - div key = { experiment.id } - className = "experiment-card" - onClick = { - () => navigate(`/experiment/${experiment.id}`) - } > - < - div className = "experiment-header-card" > - < - h3 className = "experiment-title" > { experiment.title } < /h3> { - experiment.date && ( < - span className = "experiment-date" > 📅{ - new Date(experiment.date).toLocaleDateString('en-US', { - month: 'short', - day: 'numeric' - }) - } < - /span> - ) - } < - /div> - - { - experiment.description && ( < - p className = "experiment-description" > { experiment.description } < /p> - ) - } - - < - div className = "experiment-meta" > { - experiment.github_repo && ( < - a href = { experiment.github_repo } - className = "meta-link" - onClick = { - (e) => e.stopPropagation() - } - target = "_blank" - rel = "noopener noreferrer" > - < - svg width = "12" - height = "12" - viewBox = "0 0 16 16" - fill = "currentColor" > - < - path d = "M8 0C3.58 0 0 3.58 0 8c0 3.54 2.29 6.53 5.47 7.59.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2.01.37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.59.82-2.15-.08-.2-.36-1.02.08-2.12 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.92.08 2.12.51.56.82 1.27.82 2.15 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.46.55.38A8.013 8.013 0 0016 8c0-4.42-3.58-8-8-8z" / > - < - /svg> - GitHub < - /a> - ) - } { - experiment.github_pr && ( < - a href = { experiment.github_pr } - className = "meta-link" - onClick = { - (e) => e.stopPropagation() - } - target = "_blank" - rel = "noopener noreferrer" > 🔀PR < - /a> - ) - } { - experiment.docker_image && ( < - span className = "meta-badge" - title = { experiment.docker_image } > 🐳{ experiment.docker_image.split(':').pop() } < - /span> - ) - } < - /div> < / - div > - )) - } < - /div> < / - div > < - /div> < / - div > < - /div> -); -} - -export default HomePage; \ No newline at end of file