DST Dashboard Backend and Frontend V1

.gitignore

@@ -5,6 +5,12 @@ __pycache__/
 .coverage
 htmlcov/
 
+# Node.js
+node_modules/
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+
 # Mac
 **/.DS_Store
 

.vscode/settings.json

@@ -0,0 +1,4 @@
+{
+    "python-envs.defaultEnvManager": "ms-python.python:conda",
+    "python-envs.defaultPackageManager": "ms-python.python:conda"
+}

dst_dashboard/.dockerignore

@@ -0,0 +1,59 @@
+# Python
+__pycache__
+*.py[cod]
+*$py.class
+*.so
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# Virtual environments
+venv/
+ENV/
+env/
+
+# Testing
+.pytest_cache/
+.coverage
+htmlcov/
+
+# IDE
+.vscode/
+.idea/
+*.swp
+*.swo
+
+# OS
+.DS_Store
+Thumbs.db
+
+# Git
+.git/
+.gitignore
+
+# Node (for frontend)
+node_modules/
+dst_dashboard/frontend/node_modules/
+dst_dashboard/frontend/build/
+
+# Database
+*.db
+*.sqlite
+.cache/
+
+# Logs
+*.log

dst_dashboard/Dockerfile

@@ -0,0 +1,28 @@
+FROM python:3.11-slim
+
+WORKDIR /app
+
+# Install system dependencies
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    && rm -rf /var/lib/apt/lists/*
+
+# Copy project files
+COPY pyproject.toml .
+COPY src/ ./src/
+COPY dst_dashboard/ ./dst_dashboard/
+
+# Install uv for faster dependency installation
+RUN pip install --no-cache-dir uv
+
+# Install dependencies
+RUN uv pip install --system --no-cache -r pyproject.toml
+
+# Create database directory
+RUN mkdir -p /app/data
+
+# Expose port
+EXPOSE 8000
+
+# Run the application
+CMD ["uvicorn", "dst_dashboard.main:app", "--host", "0.0.0.0", "--port", "8000"]

dst_dashboard/README.md

@@ -0,0 +1,90 @@
+# DST Dashboard
+
+Web-based experiment publication and visualization tool.
+
+## Quick Start
+
+```bash
+# Run API server
+uvicorn dst_dashboard.main:app --reload
+
+# API available at http://localhost:8000
+# Swagger UI at http://localhost:8000/api/docs
+```
+
+## Environment Variables
+
+```bash
+export DST_CONFIG_PATH=~/.cache/dst_dashboard/config.yaml  # Optional, default location
+export DST_MONGO_URI=mongodb://localhost:27017             # Optional, default location
+export DST_MONGO_DB_NAME=dst_dashboard                      # Optional, default name
+export DST_JWT_SECRET=<a real secret>                       # Required outside local dev
+```
+
+`config.yaml` only defines datasources (VictoriaLogs/Prometheus connections) - it no
+longer defines experiments. Experiments live only in MongoDB and are managed
+exclusively through the API (see below).
+
+## Managing experiments
+
+Creating, updating, and deleting experiments requires an admin bearer token.
+
+1. Get a token from the `Admin Page` (Accessible from the vaclab home page).
+2. Use it as `Authorization: Bearer <token>` on any write request below.
+
+ `title` must be unique across all experiments.
+
+### Create
+
+```bash
+TOKEN="<token>"
+
+curl -X POST https://api.dashboard.lab.vac.dev/experiments \
+  -H "Authorization: Bearer $TOKEN" \
+  -H "Content-Type: application/json" \
+  -d @experiment.json
+```
+
+The response includes the generated `id` - save it, you'll need it for update/delete.
+A `409` means an experiment with that `title` already exists.
+
+### Update
+
+```bash
+curl -X PUT https://api.dashboard.lab.vac.dev/experiments/<id> \
+  -H "Authorization: Bearer $TOKEN" \
+  -H "Content-Type: application/json" \
+  -d @experiment.json
+```
+
+Only reprocesses datasets/panels if their configuration actually changed.
+
+### Delete
+
+```bash
+curl -X DELETE https://api.dashboard.lab.vac.dev/experiments/<id> \
+  -H "Authorization: Bearer $TOKEN"
+```
+
+Cascades to the experiment's datasets and panels.
+
+### Bulk-loading from a file
+
+See `dst_dashboard/scripts/seed_experiments.py` for loading many experiments at once
+from a JSON file (`{"experiments": [...]}`, same shape as a single experiment's body).
+
+There's also a small UI for all of this in the `Admin Page` - paste JSON, create/edit/delete
+without needing curl.
+
+## Structure
+
+```
+dst_dashboard/
+├── main.py                 # FastAPI app entry
+├── auth.py                 # Admin JWT issuing/verification
+├── config/                 # Config loading + data models
+├── processors/              # Data fetching/transformation
+├── storage/                 # MongoDB client
+├── scripts/                  # Seeding/ops scripts
+└── api/                      # FastAPI REST endpoints
+```

dst_dashboard/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"

dst_dashboard/api/__init__.py

@@ -0,0 +1 @@
+"""FastAPI REST for DST Dashboard."""

dst_dashboard/api/admin.py

@@ -0,0 +1,157 @@
+"""Admin API routes for configuration and data management."""
+
+import logging
+from pathlib import Path
+
+from fastapi import APIRouter, Depends, HTTPException, Request
+from fastapi.responses import HTMLResponse
+
+from dst_dashboard.auth import create_admin_token, require_admin_token
+from dst_dashboard.config.utils import LoadConfig
+from dst_dashboard.processors.experiment_processor import ExperimentProcessor
+from dst_dashboard.storage.db import DSTDatabase
+
+router = APIRouter(prefix="/admin", tags=["admin"])
+logger = logging.getLogger(__name__)
+
+_ADMIN_HTML_PATH = Path(__file__).parent.parent / "static" / "admin.html"
+
+
+@router.post("/reload")
+def reload_config(request: Request, _: None = Depends(require_admin_token)):
+    """Reload datasources from config.yaml (experiments are API-managed and unaffected)."""
+    try:
+        logger.info("Reloading configuration...")
+
+        # Load and validate configuration
+        config = LoadConfig().WithValidateDatasources()
+        logger.info(f"Reloaded configuration with {len(config.datasources)} datasources")
+
+        # Update app state
+        request.app.state.config = config
+        request.app.state.datasources = config.datasources
+
+        # Initialize database
+        db = DSTDatabase()
+
+        # Clear and re-insert datasources
+        db.datasources.delete_many({})
+        db.insert_datasource_list(config.datasources)
+        logger.info(f"Re-inserted {len(config.datasources)} datasources")
+
+        return {
+            "status": "success",
+            "message": "Datasources reloaded successfully",
+            "summary": {
+                "datasources": len(config.datasources),
+            },
+        }
+
+    except Exception as e:
+        logger.error(f"Failed to reload configuration: {e}", exc_info=True)
+        raise HTTPException(status_code=500, detail=f"Failed to reload configuration: {str(e)}")
+
+
+@router.delete("/datasets/{experiment_id}/{dataset_name}")
+def clear_dataset(experiment_id: str, dataset_name: str, _: None = Depends(require_admin_token)):
+    """Clear cached dataset data to force re-fetch on next request."""
+    try:
+        db = DSTDatabase()
+
+        if db.delete_dataset(experiment_id, dataset_name):
+            logger.info(f"Cleared dataset: {experiment_id}:{dataset_name}")
+            return {
+                "status": "success",
+                "message": f"Dataset '{dataset_name}' cleared successfully",
+            }
+        else:
+            raise HTTPException(status_code=404, detail=f"Dataset '{dataset_name}' not found")
+
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Failed to clear dataset: {e}")
+        raise HTTPException(status_code=500, detail=f"Failed to clear dataset: {str(e)}")
+
+
+@router.delete("/datasets")
+def clear_all_datasets(_: None = Depends(require_admin_token)):
+    """Clear all cached dataset data to force re-fetch."""
+    try:
+        db = DSTDatabase()
+        count = db.clear_all_dataset_cache()
+
+        logger.info(f"Cleared {count} datasets")
+
+        return {"status": "success", "message": f"Cleared {count} datasets successfully"}
+
+    except Exception as e:
+        logger.error(f"Failed to clear datasets: {e}")
+        raise HTTPException(status_code=500, detail=f"Failed to clear datasets: {str(e)}")
+
+
+@router.post("/experiments/{experiment_id}/reprocess")
+def reprocess_experiment(
+    experiment_id: str, request: Request, _: None = Depends(require_admin_token)
+):
+    """Reprocess a single experiment - fetch datasets and regenerate panels."""
+    try:
+        db = DSTDatabase()
+
+        # Get experiment from database
+        experiment_data = db.get_experiment(experiment_id)
+        if not experiment_data:
+            raise HTTPException(status_code=404, detail="Experiment not found")
+
+        # Get config from app state
+        config = request.app.state.config
+
+        # Initialize processor
+        from dst_dashboard.config.data_structures import ExperimentConfig
+
+        processor = ExperimentProcessor(config, db)
+
+        experiment = ExperimentConfig(**experiment_data)
+
+        logger.info(f"Reprocessing experiment: {experiment_id}")
+
+        # Process the experiment (this will fetch datasets and transform panels)
+        success = processor.process_experiment(experiment)
+
+        if success:
+            return {
+                "status": "success",
+                "message": f"Experiment '{experiment_id}' reprocessed successfully",
+                "experiment_id": experiment_id,
+                "datasets_count": len(experiment.datasets),
+                "panels_count": len(experiment.panels),
+            }
+        else:
+            raise HTTPException(
+                status_code=500, detail=f"Failed to reprocess experiment '{experiment_id}'"
+            )
+
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Failed to reprocess experiment: {e}", exc_info=True)
+        raise HTTPException(status_code=500, detail=f"Failed to reprocess experiment: {str(e)}")
+
+
+@router.get("/token", response_class=HTMLResponse)
+def admin_page():
+    """
+    Serve the admin page: a "Generate token" button, plus a small UI for
+    creating/editing/deleting experiments by pasting JSON.
+
+    Not gated by require_admin_token - you need this page to get a token in
+    the first place. Access is instead controlled upstream by Authentik
+    forward-auth on this ingress path (which covers both GET and POST here).
+    """
+    return HTMLResponse(content=_ADMIN_HTML_PATH.read_text())
+
+
+@router.post("/token")
+def generate_admin_token():
+    """Mint a fresh admin-scope token. Same protection boundary as GET /admin/token."""
+    return {"token": create_admin_token()}