Merge pull request #276 from finos/feat/backend-hardening

feat(backend): security hardening + provider_api invariant + expanded API surface + alembic migrations

Author: fgogolli

.github/workflows/ci-quality.yml

@@ -35,6 +35,14 @@ on:
       - '.ruff.toml'
       - '.github/workflows/ci-quality.yml'
 
+# Cancel in-progress runs when a new push arrives on the same branch/ref.
+# The auto-format job is exempted via its own job-level concurrency block
+# (cancel-in-progress: false) because it pushes a commit back to the branch —
+# cancelling it mid-push could leave the branch in a partially formatted state.
+concurrency:
+  group: ci-${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 permissions:
   contents: read
 
@@ -91,11 +99,20 @@ jobs:
 
   auto-format:
     name: Auto-Format Code
-    # Same-repo PRs only: push commits as the ORB CICD App.
-    # Fork PRs are handled by the auto-format-suggest job below.
-    if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository
+    # Same-repo PRs only. Fork PRs handled by auto-format-suggest below.
+    # Bot-actor guard prevents the job re-triggering on its own push.
+    if: |
+      github.event_name == 'pull_request'
+      && github.event.pull_request.head.repo.full_name == github.repository
+      && github.actor != 'open-resource-broker-cicd[bot]'
     runs-on: ubuntu-latest
     needs: [config, setup-cache]
+    # Override the workflow-level concurrency so this job is never cancelled
+    # mid-run.  If it were interrupted after `ruff` rewrites files but before
+    # `git push`, the PR branch would be left in a half-formatted state.
+    concurrency:
+      group: ci-auto-format-${{ github.ref }}
+      cancel-in-progress: false
     permissions:
       contents: write
     steps:
@@ -134,7 +151,7 @@ jobs:
         run: |
           git add .
           if ! git diff --staged --quiet; then
-            git commit -m "style: auto-format code with ruff [skip ci]"
+            git commit -m "style: auto-format code with ruff"
             git push
           fi
 

.github/workflows/ci-tests.yml

@@ -22,6 +22,13 @@ on:
   schedule:
     - cron: '0 2 * * *'
 
+# Cancel in-progress runs when a new push arrives on the same branch/ref.
+# Scheduled nightly runs use a stable group key so they never cancel each
+# other (there is only ever one running at a time by schedule).
+concurrency:
+  group: ci-${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 permissions:
   contents: read
 
@@ -84,12 +91,21 @@ jobs:
       environment: ${{ needs.config.outputs.environment }}
       testing-flag: ${{ needs.config.outputs.testing-flag }}
 
-  onmoto-tests:
-    name: Onmoto Tests (Default Python)
+  # Per-provider matrix. One job per provider subtree under
+  # tests/providers/. New providers land here as new matrix entries —
+  # no separate onmoto job needed (moto-mocked tests live under
+  # tests/providers/aws/moto and run as part of the aws entry).
+  providers-tests:
+    name: Providers Tests
     needs: [config, setup-cache]
+    strategy:
+      fail-fast: false
+      matrix:
+        provider: [aws]
     uses: ./.github/workflows/reusable-test.yml
     with:
-      test-type: onmoto
+      test-type: providers
+      provider: ${{ matrix.provider }}
       python-version: ${{ needs.config.outputs.default-python-version }}
       default-python-version: ${{ needs.config.outputs.default-python-version }}
       continue-on-error: false
@@ -99,12 +115,25 @@ jobs:
       environment: ${{ needs.config.outputs.environment }}
       testing-flag: ${{ needs.config.outputs.testing-flag }}
 
-  providers-tests:
-    name: Providers & Infrastructure Tests (Default Python)
-    needs: [config, setup-cache]
+  # Serial-marked provider tests (currently the live-AWS subtree).
+  # The make target points pytest directly at ``tests/providers/<provider>/live``
+  # because ``norecursedirs`` excludes that path from auto-recursion.  Without
+  # ``--live`` the root conftest stamps every live test with ``skip_live``, so
+  # the job collects the suite, reports SKIPPED, and exits 0 — that proves the
+  # collection wiring + skip mechanism still work end-to-end on every PR.
+  # When CI is wired up with real AWS credentials, set ``PYTEST_LIVE=--live``
+  # in the workflow env to flip the suite into actual execution.
+  providers-tests-serial:
+    name: Providers Tests (Serial)
+    needs: [config, setup-cache, providers-tests]
+    strategy:
+      fail-fast: false
+      matrix:
+        provider: [aws]
     uses: ./.github/workflows/reusable-test.yml
     with:
-      test-type: providers
+      test-type: providers-serial
+      provider: ${{ matrix.provider }}
       python-version: ${{ needs.config.outputs.default-python-version }}
       default-python-version: ${{ needs.config.outputs.default-python-version }}
       continue-on-error: false
@@ -161,7 +190,7 @@ jobs:
   test-report:
     name: Generate Test Report
     runs-on: ubuntu-latest
-    needs: [config, setup-cache, tests, integration-tests, e2e-tests, onmoto-tests, providers-tests, infrastructure-tests]
+    needs: [config, setup-cache, tests, integration-tests, e2e-tests, providers-tests, providers-tests-serial, infrastructure-tests]
     if: always()
     permissions:
       contents: read

.github/workflows/reusable-test.yml

@@ -4,9 +4,14 @@ on:
   workflow_call:
     inputs:
       test-type:
-        description: 'Type of tests to run (unit, integration, e2e, performance)'
+        description: 'Type of tests to run (unit, integration, e2e, performance, providers, infrastructure)'
         required: true
         type: string
+      provider:
+        description: 'When test-type=providers, scope the run to a single provider subtree (e.g. aws, k8s).'
+        required: false
+        type: string
+        default: ''
       python-version:
         description: 'Python version to test with'
         required: false
@@ -65,7 +70,7 @@ env:
 
 jobs:
   test:
-    name: ${{ inputs.test-type == 'unit' && 'Unit Tests' || inputs.test-type == 'integration' && 'Integration Tests' || inputs.test-type == 'e2e' && 'End-to-End Tests' || format('{0} Tests', inputs.test-type) }}
+    name: ${{ inputs.test-type == 'unit' && 'Unit Tests' || inputs.test-type == 'integration' && 'Integration Tests' || inputs.test-type == 'e2e' && 'End-to-End Tests' || (inputs.test-type == 'providers' && inputs.provider != '') && format('Providers Tests ({0})', inputs.provider) || format('{0} Tests', inputs.test-type) }}
     runs-on: ${{ inputs.os }}
     permissions:
       contents: read
@@ -81,14 +86,32 @@ jobs:
           fail-on-cache-miss: false
 
       - name: Run tests
-        run: make ci-tests-${{ inputs.test-type }}
+        id: run-tests
+        run: make ci-tests-${{ inputs.test-type }} PROVIDER=${{ inputs.provider }}
         continue-on-error: ${{ inputs.continue-on-error }}
 
+      # When continue-on-error is true the overall job status stays green even
+      # if tests fail, which can hide regressions in the PR summary view.
+      # This step fires only on failure and writes a visible warning to the
+      # GitHub job summary so operators notice the failure without digging into
+      # the raw log.
+      - name: Annotate soft failure in job summary
+        if: steps.run-tests.outcome == 'failure'
+        run: |
+          echo "::warning::${{ inputs.test-type }} tests failed (continue-on-error=true). Review the test log for details."
+          {
+            echo "## :warning: ${{ inputs.test-type }} test failures"
+            echo ""
+            echo "The **${{ inputs.test-type }}** test suite reported failures."
+            echo "Overall job status is still green because \`continue-on-error: true\` is set,"
+            echo "but these failures should be investigated before merging."
+          } >> "$GITHUB_STEP_SUMMARY"
+
       - name: Upload test results
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7
         if: always()
         with:
-          name: test-results-${{ inputs.test-type }}-${{ inputs.os }}-py${{ inputs.python-version || inputs.default-python-version }}
+          name: test-results-${{ inputs.test-type }}${{ inputs.provider != '' && format('-{0}', inputs.provider) || '' }}-${{ inputs.os }}-py${{ inputs.python-version || inputs.default-python-version }}
           retention-days: 30
           path: |
             junit-*.xml
@@ -100,14 +123,14 @@ jobs:
         uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f  # v6
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
-          files: coverage-${{ inputs.test-type }}.xml
-          flags: ${{ inputs.test-type }}
-          name: ${{ inputs.test-type }}-tests
+          files: coverage-${{ inputs.test-type }}${{ inputs.provider != '' && format('-{0}', inputs.provider) || '' }}.xml
+          flags: ${{ inputs.test-type }}${{ inputs.provider != '' && format('-{0}', inputs.provider) || '' }}
+          name: ${{ inputs.test-type }}${{ inputs.provider != '' && format('-{0}', inputs.provider) || '' }}-tests
 
       - name: Upload test results to Codecov
         if: ${{ !cancelled() }}
         continue-on-error: true
         uses: codecov/test-results-action@0fa95f0e1eeaafde2c782583b36b28ad0d8c77d3  # v1
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
-          files: junit-${{ inputs.test-type }}.xml
+          files: junit-${{ inputs.test-type }}${{ inputs.provider != '' && format('-{0}', inputs.provider) || '' }}.xml

.gitignore

@@ -1,3 +1,7 @@
+.states
+.web
+reflex.lock/
+assets/external/
 # Generated files
 # pyproject.toml is now tracked - only metadata is templated, dependencies managed by Dependabot
 
@@ -279,3 +283,6 @@ coverage.json
 
 # Backup files from editors/tools
 *.bak*
+
+# Per-package coverage artifacts (never commit)
+coverage-*.xml

CHANGELOG.md

@@ -8,6 +8,28 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 <!-- insertion marker -->
 ## Unreleased
 
+### BREAKING CHANGES
+
+- **Daemon token file at `<work_dir>/server/orb-server.token`.**
+  When `orb server start` is invoked (daemon or `--foreground` mode), a
+  random bearer token is written to `<work_dir>/server/orb-server.token`
+  with mode `0600`.  The CLI reads this file to authenticate loopback admin
+  requests such as `orb server reload`.  The file is removed when the daemon
+  exits.  Operators who snapshot the work directory should exclude
+  `*.token` from backups.
+
+- **`allow_destructive_admin` config field (default `false`).**
+  Administrative endpoints that can wipe state (purge, bulk-delete, etc.)
+  now require `allow_destructive_admin: true` in the server config.  The
+  field defaults to `false` and must be opted in explicitly.
+
+- **SQL storage strategy applies Alembic migrations on startup.**
+  When `storage.strategy` is `sql`, the server now runs `alembic upgrade
+  head` automatically on startup.  Operators using a managed database
+  (RDS, Aurora) should ensure the database user has DDL privileges, or run
+  migrations out-of-band with `orb db upgrade` before starting the server
+  with `allow_auto_migrate: false`.
+
 ### Added
 
 - New `[aws]` extra (alias for AWS deps currently in core). The canonical install

CONTRIBUTING.md

@@ -330,7 +330,7 @@ The container supports multiple entry points:
 docker run --rm image --version
 
 # API server
-docker run -p 8000:8000 image system serve
+docker run -p 8000:8000 image server start --foreground
 
 # Health check
 docker run --rm image --health

MANIFEST.in

@@ -1,2 +1,3 @@
 recursive-include config *.json
 recursive-include scripts *.sh *.bat
+include src/orb/ui/rxconfig.py

README.md

@@ -229,7 +229,7 @@ See the [CLI Reference](docs/root/cli/cli-reference.md) for the full flag refere
 <details>
 <summary>REST API</summary>
 
-Example API calls. Requires `pip install "orb-py[api]"` and `orb system serve`.
+Example API calls. Requires `pip install "orb-py[api]"` and `orb server start` (add `--foreground` for an in-shell variant).
 
 ```bash
 # Get available templates