Merge pull request #682 from netzbegruenung/feat/notebook-startpage-tabs-and-stats

feat(notebook): startpage tabs, statistics & monthly keyword snapshots

Author: Movm

.github/workflows/notebook-keyword-snapshots.yml

@@ -0,0 +1,99 @@
+name: Monthly Notebook Keyword Snapshots
+
+on:
+  schedule:
+    - cron: '0 3 1 * *' # 1st of each month at 03:00 UTC
+  workflow_dispatch:
+    inputs:
+      collection_id:
+        description: 'Single collection ID to refresh (empty = all)'
+        required: false
+        type: string
+      month:
+        description: 'Override month (YYYY-MM, empty = current month)'
+        required: false
+        type: string
+
+permissions:
+  contents: read
+
+concurrency:
+  group: notebook-keyword-snapshots
+  cancel-in-progress: false
+
+jobs:
+  refresh:
+    name: Refresh keyword snapshots
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    env:
+      API_URL: ${{ vars.NOTEBOOK_API_URL || 'https://gruenerator.eu' }}
+    steps:
+      - name: Build request payload
+        id: payload
+        run: |
+          PAYLOAD='{}'
+          if [ -n "${{ inputs.collection_id }}" ]; then
+            PAYLOAD=$(jq -n --arg id "${{ inputs.collection_id }}" '{collectionId: $id}')
+          fi
+          if [ -n "${{ inputs.month }}" ]; then
+            PAYLOAD=$(echo "$PAYLOAD" | jq --arg m "${{ inputs.month }}" '. + {month: $m}')
+          fi
+          echo "payload=$PAYLOAD" >> "$GITHUB_OUTPUT"
+          echo "Payload: $PAYLOAD"
+
+      - name: Trigger snapshot refresh
+        id: refresh
+        continue-on-error: true
+        run: |
+          echo "POST $API_URL/api/internal/notebook/refresh-keywords"
+          RESULT=$(curl -sfS -X POST "$API_URL/api/internal/notebook/refresh-keywords" \
+            -H "x-admin-token: ${{ secrets.ADMIN_TOKEN }}" \
+            -H "Content-Type: application/json" \
+            -d '${{ steps.payload.outputs.payload }}' \
+            --max-time 3000)
+
+          echo "$RESULT" | jq .
+          echo "result<<EOF" >> "$GITHUB_OUTPUT"
+          echo "$RESULT" >> "$GITHUB_OUTPUT"
+          echo "EOF" >> "$GITHUB_OUTPUT"
+
+      - name: Generate summary
+        if: always()
+        run: |
+          RESULT='${{ steps.refresh.outputs.result }}'
+          RUN_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+
+          if [ -z "$RESULT" ] || [ "$RESULT" = "" ]; then
+            {
+              echo "## :x: Snapshot Refresh — No Result"
+              echo ""
+              echo "Service did not return a result. Check [workflow logs]($RUN_URL)."
+            } >> "$GITHUB_STEP_SUMMARY"
+            exit 0
+          fi
+
+          COUNT=$(echo "$RESULT" | jq -r '.count // (.results | length)')
+          SUCCESS=$(echo "$RESULT" | jq -r '.success')
+
+          if [ "$SUCCESS" = "true" ]; then
+            ICON=":white_check_mark:"
+          else
+            ICON=":x:"
+          fi
+
+          {
+            echo "## $ICON Notebook Keyword Snapshots"
+            echo ""
+            echo "**Date:** $(date -u '+%d.%m.%Y %H:%M UTC')"
+            echo "**Run:** [${{ github.run_id }}]($RUN_URL)"
+            echo "**Collections refreshed:** $COUNT"
+            echo ""
+            echo "| Collection | Keywords | Sample | Total docs | Duration (ms) |"
+            echo "|------------|---------:|-------:|-----------:|--------------:|"
+            echo "$RESULT" | jq -r '.results[] | "| \(.collectionId) | \(.keywordCount) | \(.sampleSize) | \(.totalDocuments) | \(.durationMs) |"'
+          } >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Fail if refresh errored
+        if: steps.refresh.outcome == 'failure'
+        run: exit 1

apps/api/database/postgres/migrations/add_topic_counts_to_notebook_keyword_snapshots.sql

@@ -0,0 +1,7 @@
+-- Stores monthly NLP-classified topic counts alongside the keyword snapshot.
+-- Shape: { migration: 12, klima: 4, ... } keyed by TopicCategory.
+-- Populated by the same monthly cron that fills `keywords`. Reads happen via
+-- the notebook stats endpoint so user-facing requests never hit NLP inline.
+
+ALTER TABLE notebook_keyword_snapshots
+  ADD COLUMN IF NOT EXISTS topic_counts JSONB NOT NULL DEFAULT '{}';

apps/api/database/postgres/migrations/create_notebook_keyword_snapshots.sql

@@ -0,0 +1,20 @@
+-- Monthly NLP-extracted keyword snapshots per notebook (system collection).
+-- Refreshed by a scheduled job on the 1st of each month (cron).
+-- Stats endpoint reads the latest row to render the "Häufigste Begriffe" word cloud
+-- without making an inline NLP call.
+
+CREATE TABLE IF NOT EXISTS notebook_keyword_snapshots (
+  collection_id TEXT NOT NULL,
+  month TEXT NOT NULL,                       -- 'YYYY-MM'
+  keywords JSONB NOT NULL DEFAULT '[]',      -- [{keyword, count, topic|null}]
+  total_documents INT NOT NULL DEFAULT 0,
+  sample_size INT NOT NULL DEFAULT 0,
+  computed_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+  PRIMARY KEY (collection_id, month)
+);
+
+CREATE INDEX IF NOT EXISTS idx_notebook_keyword_snapshots_month
+  ON notebook_keyword_snapshots(month DESC);
+
+CREATE INDEX IF NOT EXISTS idx_notebook_keyword_snapshots_collection_month
+  ON notebook_keyword_snapshots(collection_id, month DESC);

apps/api/routes.ts

@@ -223,6 +223,9 @@ export async function setupRoutes(app: Application): Promise<void> {
   const {
     collectionsRouter: notebookCollectionsRouter,
     interactionRouter: notebookInteractionRouter,
+    recentDocumentsRouter: notebookRecentDocumentsRouter,
+    statisticsRouter: notebookStatisticsRouter,
+    internalNotebookRouter,
   } = await import('./routes/notebook/index.js');
   const { default: nextcloudApiRouter } = await import('./routes/nextcloud/nextcloudApi.js');
   const { default: connectionsRouter } =
@@ -293,6 +296,8 @@ export async function setupRoutes(app: Application): Promise<void> {
   // the prefix, which would break the public/:token routes).
   mountNotebookContractRouter(app);
   app.use('/api/auth/notebook', authenticatedReadLimiter, notebookInteractionRouter);
+  app.use('/api/auth/notebook', authenticatedReadLimiter, notebookRecentDocumentsRouter);
+  app.use('/api/auth/notebook', authenticatedReadLimiter, notebookStatisticsRouter);
   // ts-rest contract router for /api/documents — mounts BEFORE the legacy documentsRouter
   // so ts-rest matches its own routes first; unmatched paths fall through.
   // requireAuth is applied at the prefix because all 3 contract routes require auth.
@@ -550,6 +555,7 @@ export async function setupRoutes(app: Application): Promise<void> {
   app.use('/api/internal/offboarding', offboardingRouter);
   app.use('/api/internal/gruene-api', grueneApiTestRouter);
   app.use('/api/internal/monitor', monitorInternalRouter);
+  app.use('/api/internal/notebook', internalNotebookRouter);
   app.use('/api/internal/content-sync', contentSyncRouter);
   app.use('/api/monitor', requireAuth, publicReadLimiter, monitorRouter);
 

apps/api/routes/notebook/index.ts

@@ -7,5 +7,8 @@
  */
 
 export { default as collectionsRouter } from './collectionsController.js';
+export { internalNotebookRouter } from './internalController.js';
 export { default as interactionRouter } from './interactionController.js';
+export { default as recentDocumentsRouter } from './recentDocumentsController.js';
+export { default as statisticsRouter } from './statisticsController.js';
 export * from './types.js';

apps/api/routes/notebook/internalController.ts

@@ -0,0 +1,49 @@
+import { Router, type Request, type Response } from 'express';
+
+import { requireAdminToken } from '../../middleware/adminTokenMiddleware.js';
+import {
+  refreshAllKeywordSnapshots,
+  refreshKeywordSnapshot,
+} from '../../services/notebook/notebookKeywordSnapshotService.js';
+import { toError } from '../../utils/errors/index.js';
+import { createLogger } from '../../utils/logger.js';
+
+const log = createLogger('notebook-internal');
+const router: Router = Router();
+
+/**
+ * Refresh notebook keyword snapshots. Called monthly by GitHub Actions cron
+ * (apps/api isn't running an in-process scheduler).
+ *
+ * POST /api/internal/notebook/refresh-keywords
+ *   body: { collectionId?: string, month?: 'YYYY-MM' }
+ *   - no body → refresh all system collections for current month
+ *   - { collectionId } → refresh just that one
+ *   - { month } → store under that month label (defaults to current)
+ */
+router.post(
+  '/refresh-keywords',
+  requireAdminToken,
+  async (req: Request, res: Response): Promise<void> => {
+    const body = (req.body ?? {}) as { collectionId?: string; month?: string };
+    const month = typeof body.month === 'string' && body.month ? body.month : undefined;
+
+    try {
+      if (typeof body.collectionId === 'string' && body.collectionId) {
+        log.info(`Single-collection refresh: ${body.collectionId} (month=${month ?? 'current'})`);
+        const result = await refreshKeywordSnapshot(body.collectionId, month);
+        res.json({ success: true, results: result ? [result] : [] });
+        return;
+      }
+
+      log.info(`Bulk refresh for all system collections (month=${month ?? 'current'})`);
+      const results = await refreshAllKeywordSnapshots(month);
+      res.json({ success: true, count: results.length, results });
+    } catch (error) {
+      log.error(`Snapshot refresh failed: ${toError(error).message}`);
+      res.status(500).json({ error: toError(error).message });
+    }
+  }
+);
+
+export const internalNotebookRouter = router;