bcgov · alex-struk · Jun 23, 2026 · May 20, 2026 · May 20, 2026 · May 20, 2026
@@ -5,10 +5,10 @@
 # ── Database (shared by backend-services and temporal worker) ─────────────────
 DATABASE_URL="postgresql://postgres:postgres@localhost:5432/ai_doc_intelligence?schema=public"
 # Database Connection Pool
-# Maximum number of database connections per pod (default: 5)
+# Maximum number of database connections per pod (default: 10)
 # Scale this based on: concurrent requests per pod, pod count, and Postgres max_connections (default: 100)
-# Example: 3 backend pods * 5 connections = 15 total connections
-DB_POOL_MAX=5
+# Example: 5 backend pods * 10 connections + 4 worker pods * 3 = 62 total connections
+DB_POOL_MAX=10
 
 # ── MinIO (object storage) ────────────────────────────────────────────────────
 MINIO_ROOT_USER=minioadmin

@@ -443,6 +443,25 @@ jobs:
             "app.kubernetes.io/instance=${INSTANCE_NAME}" --overwrite \
             -n "${OC_NAMESPACE}" || true
 
+      - name: Delete immutable PLG StatefulSets before upgrade
+        if: env.DEPLOY_PLG == 'true'
+        env:
+          INSTANCE_NAME: ${{ needs.metadata.outputs.instance-name }}
+          OC_NAMESPACE: ${{ needs.metadata.outputs.namespace || secrets.OPENSHIFT_NAMESPACE }}
+        run: |
+          set -euo pipefail
+          PLG_RELEASE="${INSTANCE_NAME}-plg"
+          # StatefulSet volumeClaimTemplates are immutable in Kubernetes.
+          # Delete the StatefulSets so Helm can recreate them with the correct
+          # spec. PVCs are not deleted and data is preserved.
+          for ss in loki prometheus; do
+            SS_NAME="${PLG_RELEASE}-${ss}"
+            if oc get statefulset "${SS_NAME}" -n "${OC_NAMESPACE}" &>/dev/null; then
+              echo "Deleting StatefulSet ${SS_NAME} for clean upgrade..."
+              oc delete statefulset "${SS_NAME}" -n "${OC_NAMESPACE}" --cascade=orphan
+            fi
+          done
+
       - name: Deploy PLG monitoring stack
         if: env.DEPLOY_PLG == 'true'
         env:
@@ -482,7 +501,7 @@ jobs:
             --set "alertmanager.ches.adapterUrl=http://${PLG_RELEASE}-ches-adapter:3003/" \
             --set "chesAdapter.secretName=${INSTANCE_NAME}-ches-adapter-secrets" \
             --set "alertmanager.teams.webhookUrl=${ALERTMANAGER_TEAMS_WEBHOOK_URL:-placeholder}" \
-            --wait --timeout 120s
+            --wait --timeout 300s
 
       - name: Restart deployments and wait
         env:

@@ -44,7 +44,7 @@ MANAGE_WORKER=true npm run test:int:workflow
 npm run test:int:workflow:with-worker
 
 # Run with specific template and test file
-WORKFLOW_TEMPLATE=multi-page-report-workflow TEST_FILE=multi-page-sample-1.pdf npm run test:int:workflow:with-worker
+WORKFLOW_SLUG=multi-page-report TEST_FILE=multi-page-sample-1.pdf npm run test:int:workflow:with-worker
 ```
 
 **Benefits:**
@@ -64,7 +64,7 @@ npm run dev
 
 # Terminal 2: Run the test
 cd ~/GitHub/ai-adoption-document-intelligence/apps/backend-services
-WORKFLOW_TEMPLATE=multi-page-report-workflow TEST_FILE=multi-page-sample-1.pdf npm run test:int:workflow
+WORKFLOW_SLUG=multi-page-report TEST_FILE=multi-page-sample-1.pdf npm run test:int:workflow
 ```
 
 **Benefits:**
@@ -78,7 +78,8 @@ WORKFLOW_TEMPLATE=multi-page-report-workflow TEST_FILE=multi-page-sample-1.pdf n
 |----------|---------|-------------|
 | `MANAGE_WORKER` | `false` | Set to `true` to have the test start/stop the worker |
 | `WORKER_STARTUP_DELAY` | `5000` | Milliseconds to wait for worker to initialize (when `MANAGE_WORKER=true`) |
-| `WORKFLOW_TEMPLATE` | `standard-ocr-workflow` | Workflow template to test (from `docs-md/templates/`) |
+| `WORKFLOW_SLUG` | `standard-ocr` | Seeded workflow lineage slug (`workflow_versions` in DB) |
+| `WORKFLOW_VERSION` | *(head)* | Optional pin to a specific `version_number` |
 | `TEST_FILE` | `test-document.jpg` | Test file to upload (from `integration-tests/`) |
 | `BACKEND_URL` | `http://localhost:3002` | Backend API URL |
 | `TEMPORAL_ADDRESS` | `localhost:7233` | Temporal server address |
@@ -125,7 +126,7 @@ Example output:
 
 ```bash
 # Test with managed worker and custom template
-MANAGE_WORKER=true WORKFLOW_TEMPLATE=multi-page-report-workflow TEST_FILE=multi-page-sample-1.pdf npm run test:int:workflow
+MANAGE_WORKER=true WORKFLOW_SLUG=multi-page-report TEST_FILE=multi-page-sample-1.pdf npm run test:int:workflow
 
 # Test with longer timeout for complex workflows
 MANAGE_WORKER=true TEST_TIMEOUT=600000 npm run test:int:workflow

@@ -43,14 +43,33 @@ npm run test:int:workflow
 ## What the Test Does
 
 1. ✓ Checks that Temporal (port 7233) and Backend (port 3002) are running
-2. ✓ Loads `docs-md/templates/standard-ocr-workflow.json`
+2. ✓ Resolves a workflow by `workflow_slug` and uses its `workflowVersionId`
 3. ✓ Loads test image from `integration-tests/test-document.jpg`
-4. ✓ Creates workflow config in database
-5. ✓ Uploads test document
+4. ✓ Uploads test document (which triggers versionId-only Temporal start)
 6. ✓ Monitors workflow execution through Temporal
 7. ✓ Shows real-time progress of each activity
 8. ✓ Cleans up test data when done
 
+## Troubleshooting
+
+### Document status `failed` immediately (no Temporal workflow)
+
+Upload returns before background OCR runs. If the document moves to `failed` without a `workflow_execution_id`, OCR never reached Temporal.
+
+**Common cause:** API key `group_id` contains characters invalid for blob paths (e.g. `seed-default-group`). OCR reads blobs via `validateBlobFilePath`, which requires group ids matching `/^[a-z][0-9a-z]+$/`.
+
+**Fix:** Re-seed and point the API key at the seeded group:
+
+```bash
+cd apps/backend-services
+npm run db:seed
+# Then update api_keys.group_id to seeddefaultgroup for your test key
+```
+
+### `workflow not found` in Temporal
+
+Ensure the Temporal worker is running (`cd apps/temporal && npm run dev`) and listening on task queue `ocr-processing` (default). The test now polls the document API for `workflow_execution_id` before monitoring Temporal.
+
 ## Expected Behavior
 
 The test will run and **should currently fail** at the `azureOcr.submit` activity with:

@@ -49,7 +49,8 @@ The test uses the following environment variables (with defaults):
 - `TEMPORAL_NAMESPACE` (default: `default`)
 - `TEST_API_KEY` (required for authentication)
 - `TEST_TIMEOUT` (default: `300000` ms / 5 minutes)
-- `WORKFLOW_TEMPLATE` (default: `standard-ocr-workflow`)
+- `WORKFLOW_SLUG` (default: `standard-ocr`) — seeded workflow lineage slug in the database
+- `WORKFLOW_VERSION` (optional) — pin a specific `version_number`; default is head version
 - `TEST_FILE` (default: `test-document.jpg`)
 
 These are typically already configured in your `.env` file.
@@ -81,17 +82,21 @@ ts-node -r tsconfig-paths/register integration-tests/test-graph-workflow.ts
 
 ### Testing Different Workflows
 
-You can test different workflow templates by setting the `WORKFLOW_TEMPLATE` environment variable:
+The harness resolves a **seeded** workflow by `WORKFLOW_SLUG` (not by loading JSON from disk). Re-seed after template changes:
 
 ```bash
-# Test standard OCR workflow (default)
+cd apps/backend-services && npx tsx ../shared/prisma/seed.ts
+```
+
+```bash
+# Test standard OCR workflow (default slug: standard-ocr)
 npm run test:int:workflow
 
 # Test multi-page report workflow
-WORKFLOW_TEMPLATE=multi-page-report-workflow npm run test:int:workflow
+WORKFLOW_SLUG=multi-page-report npm run test:int:workflow
 
 # Test with a different file
-TEST_FILE=my-test-file.pdf WORKFLOW_TEMPLATE=multi-page-report-workflow npm run test:int:workflow
+TEST_FILE=my-test-file.pdf WORKFLOW_SLUG=multi-page-report npm run test:int:workflow
 ```
 
 ## Test Flow
@@ -102,10 +107,9 @@ TEST_FILE=my-test-file.pdf WORKFLOW_TEMPLATE=multi-page-report-workflow npm run
    └─ Check Backend API health endpoint
 
 2. Test Setup
-   ├─ Load workflow config from docs-md/templates/{WORKFLOW_TEMPLATE}.json
+   ├─ Resolve workflow version by WORKFLOW_SLUG (seeded workflow_versions row)
    ├─ Load test file (from integration-tests/{TEST_FILE})
-   ├─ Create workflow configuration in database
-   └─ Upload document via /api/upload
+   └─ Upload document via /api/upload (workflow_config_id → versionId-only Temporal start)
 
 3. Workflow Execution
    ├─ Initialize Temporal client
@@ -177,19 +181,21 @@ When running successfully, you'll see output like:
 - Ensure backend is running: `cd apps/backend-services && npm run start:dev`
 - Check health endpoint: `curl http://localhost:3001/health`
 
-### "Workflow template not found"
+### "Workflow not found" / unknown slug
 
-- Verify the template exists: `ls docs-md/templates/standard-ocr-workflow.json`
+- Re-seed: `npx tsx ../shared/prisma/seed.ts` from `apps/backend-services`
+- List slugs in DB or check seed: `standard-ocr`, `multi-page-report`
+- Canonical JSON templates: `docs-md/graph-workflows/templates/standard-ocr-workflow.json`
 
 ### "Test file not found"
 
 - Verify the test file exists: `ls apps/backend-services/integration-tests/test-document.jpg`
 
 ## Test Data
 
-- **Workflow Templates**:
-  - `docs-md/templates/standard-ocr-workflow.json` (default)
-  - `docs-md/templates/multi-page-report-workflow.json`
+- **Workflow slugs (seed)**:
+  - `standard-ocr` (default) — template `docs-md/graph-workflows/templates/standard-ocr-workflow.json`
+  - `multi-page-report` — template `docs-md/graph-workflows/templates/multi-page-report-workflow.json`
 - **Test Documents**:
   - `apps/backend-services/integration-tests/test-document.jpg` (default)
   - You can add your own test files and reference them via `TEST_FILE` env var

@@ -9,7 +9,9 @@
 set -e
 
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-PROJECT_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)"
+# script lives at apps/backend-services/integration-tests/graph-workflow-tests
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/../../../.." && pwd)"
+BACKEND_DIR="$PROJECT_ROOT/apps/backend-services"
 
 # Colors
 RED='\033[0;31m'
@@ -24,15 +26,18 @@ echo -e "${BLUE}===========================================================${NC}
 echo ""
 
 # Check if services are running
-check_service() {
+check_http_service() {
     local name=$1
     local url=$2
     local max_attempts=3
     local attempt=1
 
     while [ $attempt -le $max_attempts ]; do
-        if curl -s -f -o /dev/null "$url"; then
-            echo -e "${GREEN}✓${NC} $name is running"
+        # Backend health check: 200 or 401 both indicate the server is up.
+        local status
+        status=$(curl -s -o /dev/null -w "%{http_code}" "$url" || true)
+        if [ "$status" = "200" ] || [ "$status" = "401" ]; then
+            echo -e "${GREEN}✓${NC} $name is running (HTTP $status)"
             return 0
         fi
         attempt=$((attempt + 1))
@@ -45,18 +50,33 @@ check_service() {
     return 1
 }
 
+# Temporal gRPC endpoint is not HTTP; just check the TCP port is open.
+check_tcp_port() {
+    local name=$1
+    local host=$2
+    local port=$3
+    if command -v nc >/dev/null 2>&1; then
+        if nc -z "$host" "$port" >/dev/null 2>&1; then
+            echo -e "${GREEN}✓${NC} $name is listening on $host:$port"
+            return 0
+        fi
+    fi
+    echo -e "${YELLOW}⚠${NC} Could not verify $name port ($host:$port). Install netcat (nc) to enable this check."
+    return 0
+}
+
 # Service checks
 echo "Checking required services..."
 echo ""
 
 TEMPORAL_OK=false
 BACKEND_OK=false
 
-if check_service "Temporal Server" "http://localhost:7233"; then
+if check_tcp_port "Temporal Server" "localhost" "7233"; then
     TEMPORAL_OK=true
 fi
 
-if check_service "Backend API" "http://localhost:3002/api/models"; then
+if check_http_service "Backend API" "http://localhost:3002/api/models"; then
     BACKEND_OK=true
 fi