feat(hub): API docs sidebar group with hub-proxied Swagger UIs

Adds a new "API docs" sidebar group linking to the crawler + git-metadata-extractor Swagger UIs, routed through the hub at /api/crawler/docs and /api/extractor/docs so discovery is gated by HUB_AUTH instead of being anonymously reachable on the upstream ports. The proxied openapi.json rewrites the `servers` field to the public upstream URL so Swagger UI's "Try it out" still works once the user pastes their CRAWLER_API_TOKEN / EXTRACTOR_API_TOKEN into Authorize.

User-facing URLs derive from a single HUB_PUBLIC_HOST env var (default `localhost`); per-service overrides still win. Sidebar links are tightened to a uniform 28px height — dashboards collapsed from a 2-line summary to a single-line row with the summary moved into the link's title= tooltip.

Author: caviri

infra/.env.example

@@ -187,3 +187,14 @@ GITHUB_API_TOKEN=
 GITLAB_API_TOKEN=
 # Optional: separate PAT for the grimoire intake cron sidecar.
 # GRIMOIRE_GITHUB_TOKEN=
+
+# Public hostname users hit in the browser. The hub builds every
+# HUB_*_BROWSER_URL and HUB_*_DOCS_URL default by gluing this onto the
+# host-published port. Override an individual URL below only if a service
+# lives on a different host or sits behind a reverse proxy.
+# HUB_PUBLIC_HOST=openpulse.epfl.ch
+# HUB_NEO4J_BROWSER_URL=
+# HUB_SPARQL_BROWSER_URL=
+# HUB_OPENSEARCH_DASHBOARDS_URL=
+# HUB_CRAWLER_DOCS_URL=
+# HUB_EXTRACTOR_DOCS_URL=

infra/open-pulse-stack/docker-compose.yml

@@ -329,7 +329,10 @@ services:
       HUB_APPLIER_URL: "${HUB_APPLIER_URL:-http://projects-applier:8000}"
       HUB_SPARQL_URL: "${HUB_SPARQL_URL:-http://sparql-proxy:7878}"
       HUB_NEO4J_URL: "${HUB_NEO4J_URL:-bolt://neo4j:7687}"
-      HUB_KIBITER_URL: "${HUB_KIBITER_URL:-http://localhost:7508}"
+      HUB_KIBITER_URL: "${HUB_KIBITER_URL:-http://${HUB_PUBLIC_HOST:-localhost}:7508}"
+      # Single host knob the URL defaults below derive from. Override individual
+      # HUB_*_BROWSER_URL / HUB_*_DOCS_URL values to bypass this.
+      HUB_PUBLIC_HOST: "${HUB_PUBLIC_HOST:-localhost}"
       # OpenSearch (used by /api/databases/opensearch/* and the marquee stats)
       HUB_OPENSEARCH_URL: "${HUB_OPENSEARCH_URL:-https://opensearch-node1:9200}"
       HUB_OPENSEARCH_USERNAME: "${HUB_OPENSEARCH_USERNAME:-${OPENSEARCH_USERNAME:-admin}}"
@@ -346,11 +349,24 @@ services:
       SPARQL_AUTH: "${SPARQL_AUTH:-}"
       # Used by /api/projects/build-by-owner to query Neo4j directly.
       NEO4J_AUTH: "${NEO4J_AUTH:-}"
-      # User-facing dashboard URLs (host-published ports / external links)
-      HUB_NEO4J_BROWSER_URL: "${HUB_NEO4J_BROWSER_URL:-http://localhost:7474}"
-      HUB_SPARQL_BROWSER_URL: "${HUB_SPARQL_BROWSER_URL:-http://localhost:7878}"
-      HUB_OPENSEARCH_DASHBOARDS_URL: "${HUB_OPENSEARCH_DASHBOARDS_URL:-http://localhost:5601}"
+      # User-facing dashboard URLs (host-published ports / external links).
+      # Default to HUB_PUBLIC_HOST so an operator only has to set one var;
+      # set an explicit URL here to override on a per-service basis.
+      HUB_NEO4J_BROWSER_URL: "${HUB_NEO4J_BROWSER_URL:-http://${HUB_PUBLIC_HOST:-localhost}:7503}"
+      HUB_SPARQL_BROWSER_URL: "${HUB_SPARQL_BROWSER_URL:-http://${HUB_PUBLIC_HOST:-localhost}:7502}"
+      HUB_OPENSEARCH_DASHBOARDS_URL: "${HUB_OPENSEARCH_DASHBOARDS_URL:-http://${HUB_PUBLIC_HOST:-localhost}:7508}"
       HUB_ONTOLOGY_URL: "${HUB_ONTOLOGY_URL:-https://github.com/sdsc-ordes/open-pulse-ontology}"
+      # Swagger UIs are proxied through the hub (auth-gated) by default, so
+      # the upstream pages don't need to be reachable anonymously. Set a full
+      # URL to bypass the hub proxy.
+      HUB_CRAWLER_DOCS_URL: "${HUB_CRAWLER_DOCS_URL:-/api/crawler/docs}"
+      HUB_EXTRACTOR_DOCS_URL: "${HUB_EXTRACTOR_DOCS_URL:-/api/extractor/docs}"
+      # Public URL Swagger UI's "Try it out" hits — used in the rewritten
+      # `servers` field. Defaults to ${HUB_PUBLIC_HOST}:${PORT}.
+      HUB_CRAWLER_PUBLIC_URL: "${HUB_CRAWLER_PUBLIC_URL:-}"
+      HUB_EXTRACTOR_PUBLIC_URL: "${HUB_EXTRACTOR_PUBLIC_URL:-}"
+      # Used by the openapi proxy to fetch the spec in-network.
+      HUB_EXTRACTOR_URL: "${HUB_EXTRACTOR_URL:-http://git-metadata-extractor:1234}"
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
       - ${OPEN_PULSE_DATA_DIR:-./data}/hub:/data/hub

src/open_pulse/gui/hub/config.py

@@ -56,6 +56,13 @@ class Settings:
     ontology_url: str
     """External URL of the open-pulse ontology documentation."""
 
+    crawler_docs_url: str
+    """User-facing URL of the crawler's OpenAPI / Swagger docs (host-published)."""
+
+    extractor_docs_url: str
+    """User-facing URL of the git-metadata-extractor's Swagger docs
+    (host-published)."""
+
     opensearch_url: str
     """Base URL of OpenSearch (in-network)."""
 
@@ -115,13 +122,20 @@ def load_settings() -> Settings:
             "(e.g. python -c 'import secrets; print(secrets.token_urlsafe(32))')."
         )
 
+    # User-facing hostname for the host-published service ports. The
+    # individual HUB_*_BROWSER_URL / HUB_*_DOCS_URL overrides still win;
+    # this is just the default so an operator only has to set one var.
+    public_host = os.environ.get("HUB_PUBLIC_HOST", "localhost").strip() or "localhost"
+
     return Settings(
         auth_token=auth,
         data_dir=Path(os.environ.get("HUB_DATA_DIR", "/data/hub")),
         applier_url=os.environ.get("HUB_APPLIER_URL", "http://projects-applier:8000"),
         sparql_url=os.environ.get("HUB_SPARQL_URL", "http://sparql-proxy:7878"),
         neo4j_url=os.environ.get("HUB_NEO4J_URL", "bolt://neo4j:7687"),
-        grimoire_kibiter_url=os.environ.get("HUB_KIBITER_URL", "http://localhost:7508"),
+        grimoire_kibiter_url=os.environ.get(
+            "HUB_KIBITER_URL", f"http://{public_host}:7508"
+        ),
         opensearch_url=os.environ.get(
             "HUB_OPENSEARCH_URL", "https://opensearch-node1:9200"
         ),
@@ -133,17 +147,25 @@ def load_settings() -> Settings:
         ),
         opensearch_verify_tls=_env_bool("HUB_OPENSEARCH_VERIFY_TLS", default=False),
         neo4j_browser_url=os.environ.get(
-            "HUB_NEO4J_BROWSER_URL", "http://localhost:7474"
+            "HUB_NEO4J_BROWSER_URL", f"http://{public_host}:7503"
         ),
         sparql_browser_url=os.environ.get(
-            "HUB_SPARQL_BROWSER_URL", "http://localhost:7878"
+            "HUB_SPARQL_BROWSER_URL", f"http://{public_host}:7502"
         ),
         opensearch_dashboards_url=os.environ.get(
-            "HUB_OPENSEARCH_DASHBOARDS_URL", "http://localhost:5601"
+            "HUB_OPENSEARCH_DASHBOARDS_URL", f"http://{public_host}:7508"
         ),
         ontology_url=os.environ.get(
             "HUB_ONTOLOGY_URL", "https://github.com/sdsc-ordes/open-pulse-ontology"
         ),
+        # Default to the hub-proxied Swagger surfaces (routes/crawler.py +
+        # routes/extractor.py) — hub auth gates discovery, so we don't have
+        # to expose the upstream Swagger pages anonymously on their public
+        # ports. Override with a full URL to bypass the proxy.
+        crawler_docs_url=os.environ.get("HUB_CRAWLER_DOCS_URL", "/api/crawler/docs"),
+        extractor_docs_url=os.environ.get(
+            "HUB_EXTRACTOR_DOCS_URL", "/api/extractor/docs"
+        ),
         applier_auth=os.environ.get("APPLIER_AUTH", "").strip(),
         sparql_user=(_parse_user_pass(os.environ.get("SPARQL_AUTH", ""))[0]),
         sparql_password=(_parse_user_pass(os.environ.get("SPARQL_AUTH", ""))[1]),

src/open_pulse/gui/hub/main.py

@@ -13,7 +13,17 @@
 from fastapi.templating import Jinja2Templates
 
 from .auth import _COOKIE_NAME, clear_session, get_settings, require_auth
-from .routes import admin, crawler, databases, pipeline, projects, services, stack, stats
+from .routes import (
+    admin,
+    crawler,
+    databases,
+    extractor,
+    pipeline,
+    projects,
+    services,
+    stack,
+    stats,
+)
 
 _HERE = Path(__file__).parent
 log = logging.getLogger(__name__)
@@ -64,6 +74,21 @@ async def _lifespan(app: FastAPI):
     },
 ]
 templates.env.globals["ontology_url"] = _settings.ontology_url
+# Swagger UIs for the two pipeline services that ship an HTTP API. Rendered
+# as their own compact sidebar group so users can poke the endpoints
+# directly without digging through Portainer.
+templates.env.globals["api_docs"] = [
+    {
+        "name": "Crawler",
+        "tech": "FastAPI",
+        "url": _settings.crawler_docs_url,
+    },
+    {
+        "name": "Metadata extractor",
+        "tech": "FastAPI",
+        "url": _settings.extractor_docs_url,
+    },
+]
 
 app.include_router(services.router)
 app.include_router(projects.router)
@@ -72,6 +97,7 @@ async def _lifespan(app: FastAPI):
 app.include_router(stack.router)
 app.include_router(stats.router)
 app.include_router(crawler.router)
+app.include_router(extractor.router)
 app.include_router(admin.router)
 
 

src/open_pulse/gui/hub/routes/crawler.py

@@ -8,11 +8,14 @@
 
 from __future__ import annotations
 
+import json
 import os
 from typing import Any
 
 import httpx
-from fastapi import APIRouter, Depends, HTTPException
+from fastapi import APIRouter, Depends, HTTPException, Response
+from fastapi.openapi.docs import get_swagger_ui_html
+from fastapi.responses import HTMLResponse
 
 from ..auth import require_auth
 
@@ -68,6 +71,52 @@ def _passthrough(method: str, path: str, **kw: Any) -> dict[str, Any]:
     return resp.json() if resp.content else {}
 
 
+def _crawler_public_url() -> str:
+    """External base URL Swagger UI's "Try it out" hits.
+
+    Defaults to ``http://${HUB_PUBLIC_HOST}:${CRAWLER_PORT}`` so a single
+    knob keeps everything pointing at the right host. Override with
+    ``HUB_CRAWLER_PUBLIC_URL`` if the crawler sits behind a different
+    proxy (e.g. https + path-prefix).
+    """
+    explicit = os.environ.get("HUB_CRAWLER_PUBLIC_URL", "").strip()
+    if explicit:
+        return explicit.rstrip("/")
+    host = os.environ.get("HUB_PUBLIC_HOST", "localhost").strip() or "localhost"
+    port = os.environ.get("CRAWLER_PORT", "8000").strip() or "8000"
+    return f"http://{host}:{port}"
+
+
+@router.get("/docs", include_in_schema=False, dependencies=[Depends(require_auth)])
+def crawler_docs() -> HTMLResponse:
+    """Hub-gated Swagger UI for the crawler API. Reads the spec from
+    ``/api/crawler/openapi.json`` (also hub-gated). Authentication is the
+    user's hub session — no upstream token needed to *view* the surface.
+    """
+    return get_swagger_ui_html(
+        openapi_url="/api/crawler/openapi.json",
+        title="Crawler API — via Open Pulse Hub",
+    )
+
+
+@router.get(
+    "/openapi.json", include_in_schema=False, dependencies=[Depends(require_auth)]
+)
+def crawler_openapi() -> Response:
+    """Proxy the upstream spec and rewrite ``servers`` to the crawler's
+    public URL so Swagger UI's "Try it out" hits the upstream directly
+    (the user pastes ``CRAWLER_API_TOKEN`` into the Authorize dialog).
+    """
+    try:
+        upstream = httpx.get(f"{_crawler_base()}/api/v1/openapi.json", timeout=5.0)
+        upstream.raise_for_status()
+    except httpx.HTTPError as exc:
+        raise HTTPException(status_code=502, detail=str(exc)) from exc
+    spec = upstream.json()
+    spec["servers"] = [{"url": _crawler_public_url()}]
+    return Response(content=json.dumps(spec), media_type="application/json")
+
+
 @router.get("/jobs", dependencies=[Depends(require_auth)])
 def list_jobs() -> dict[str, Any]:
     return _passthrough("GET", "/api/v1/jobs")

src/open_pulse/gui/hub/routes/extractor.py

@@ -0,0 +1,66 @@
+"""Hub-gated proxy for the git-metadata-extractor (GME) Swagger UI.
+
+GME ships its OpenAPI spec at ``/openapi.json`` and Swagger UI at
+``/docs``. The upstream API endpoints are bearer-token protected, but
+the docs themselves are public on the published port. Proxying through
+the hub means hub-authenticated users can discover the API surface
+without us also having to expose the upstream port to the world.
+
+Only the read-only docs surface is proxied here — Swagger UI's
+"Try it out" is wired to hit the upstream public URL directly (with the
+user's GME bearer pasted into Authorize), so we don't need to forward
+every API path through the hub.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+
+import httpx
+from fastapi import APIRouter, Depends, HTTPException, Response
+from fastapi.openapi.docs import get_swagger_ui_html
+from fastapi.responses import HTMLResponse
+
+from ..auth import require_auth
+
+router = APIRouter(prefix="/api/extractor", tags=["extractor"])
+
+
+def _extractor_base() -> str:
+    """In-network URL of the GME container."""
+    return os.environ.get(
+        "HUB_EXTRACTOR_URL", "http://git-metadata-extractor:1234"
+    ).rstrip("/")
+
+
+def _extractor_public_url() -> str:
+    """External base URL Swagger UI's "Try it out" hits."""
+    explicit = os.environ.get("HUB_EXTRACTOR_PUBLIC_URL", "").strip()
+    if explicit:
+        return explicit.rstrip("/")
+    host = os.environ.get("HUB_PUBLIC_HOST", "localhost").strip() or "localhost"
+    port = os.environ.get("EXTRACTOR_PORT", "1234").strip() or "1234"
+    return f"http://{host}:{port}"
+
+
+@router.get("/docs", include_in_schema=False, dependencies=[Depends(require_auth)])
+def extractor_docs() -> HTMLResponse:
+    return get_swagger_ui_html(
+        openapi_url="/api/extractor/openapi.json",
+        title="Metadata extractor API — via Open Pulse Hub",
+    )
+
+
+@router.get(
+    "/openapi.json", include_in_schema=False, dependencies=[Depends(require_auth)]
+)
+def extractor_openapi() -> Response:
+    try:
+        upstream = httpx.get(f"{_extractor_base()}/openapi.json", timeout=5.0)
+        upstream.raise_for_status()
+    except httpx.HTTPError as exc:
+        raise HTTPException(status_code=502, detail=str(exc)) from exc
+    spec = upstream.json()
+    spec["servers"] = [{"url": _extractor_public_url()}]
+    return Response(content=json.dumps(spec), media_type="application/json")

src/open_pulse/gui/hub/static/app.css

@@ -183,7 +183,7 @@ body {
   border-top: 1px solid var(--border);
 }
 .sidebar-group-label {
-  padding: 10px 12px 4px;
+  padding: 8px 10px 2px;
   font-size: 10px;
   font-weight: 600;
   text-transform: uppercase;
@@ -199,16 +199,19 @@ body {
   flex-shrink: 0;
 }
 .sidebar-link-ext:hover .ext-arrow { color: var(--accent); }
-/* Dashboard rows are taller — they include a tech badge + a one-line
-   summary so users know which surface is which. */
-.sidebar-link-dash {
-  align-items: flex-start;
-  padding-top: 10px;
-  padding-bottom: 10px;
-}
-.sidebar-link-dash .dash-meta { flex: 1; min-width: 0; }
-.sidebar-link-dash .dash-head { display: flex; align-items: baseline; gap: 6px; }
-.sidebar-link-dash .dash-name { font-weight: 600; color: var(--fg); }
+/* Dashboard + API-docs rows: single-line, same height as a regular
+   sidebar-link. The tech badge sits inline after the name; the full
+   summary moves into the row's title= tooltip so users can hover for
+   detail without the link itself growing two lines tall. */
+.sidebar-link-dash .dash-name {
+  font-weight: 600;
+  color: var(--fg);
+  flex-shrink: 1;
+  min-width: 0;
+  overflow: hidden;
+  text-overflow: ellipsis;
+  white-space: nowrap;
+}
 .sidebar-link-dash .dash-tech {
   font-size: 9px;
   text-transform: uppercase;
@@ -217,20 +220,16 @@ body {
   border-radius: 3px;
   background: var(--accent-soft);
   color: var(--accent);
-}
-.sidebar-link-dash .dash-summary {
-  font-size: 11px;
-  color: var(--fg-faint);
-  margin-top: 1px;
-  line-height: 1.35;
-  white-space: normal;
+  flex-shrink: 0;
 }
 .sidebar-nav { overflow-y: auto; }    /* let the nav scroll if 3 groups overflow */
 .sidebar-link {
-  display: flex; align-items: center; gap: 10px;
-  padding: 8px 12px;
-  border-radius: 8px;
-  font-size: 13px; font-weight: 500;
+  display: flex; align-items: center; gap: 9px;
+  padding: 5px 10px;
+  border-radius: 6px;
+  font-size: 12.5px; font-weight: 500;
+  line-height: 1.25;
+  min-height: 28px;
   color: var(--fg-muted);
   text-decoration: none;
   border: 1px solid transparent;