Skip to content

Commit 070be4e

Browse files
fgogollifgogolli
committed
ci,build: parallel test matrix + serial provider job + concurrency cancellation
- Unit / integration / infrastructure / providers via pytest-xdist (-n 2 --dist=loadscope) - providers-tests-serial job for @pytest.mark.serial tests on every PR - e2e job pairs continue-on-error with GITHUB_STEP_SUMMARY failure annotation - Concurrency: fresh push cancels in-flight runs (auto-format job opts out) - Docker compose + entrypoint aligned with new server_daemon lifecycle - OpenAPI spec export via dev-tools - .gitignore: per-package coverage-*.xml artifacts
1 parent a494593 commit 070be4e

9 files changed

Lines changed: 155 additions & 38 deletions

File tree

‎.github/workflows/ci-quality.yml‎

Lines changed: 21 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,14 @@ on:
3535
- '.ruff.toml'
3636
- '.github/workflows/ci-quality.yml'
3737

38+
# Cancel in-progress runs when a new push arrives on the same branch/ref.
39+
# The auto-format job is exempted via its own job-level concurrency block
40+
# (cancel-in-progress: false) because it pushes a commit back to the branch —
41+
# cancelling it mid-push could leave the branch in a partially formatted state.
42+
concurrency:
43+
group: ci-${{ github.workflow }}-${{ github.ref }}
44+
cancel-in-progress: true
45+
3846
permissions:
3947
contents: read
4048

@@ -91,11 +99,20 @@ jobs:
9199

92100
auto-format:
93101
name: Auto-Format Code
94-
# Same-repo PRs only: push commits as the ORB CICD App.
95-
# Fork PRs are handled by the auto-format-suggest job below.
96-
if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository
102+
# Same-repo PRs only. Fork PRs handled by auto-format-suggest below.
103+
# Bot-actor guard prevents the job re-triggering on its own push.
104+
if: |
105+
github.event_name == 'pull_request'
106+
&& github.event.pull_request.head.repo.full_name == github.repository
107+
&& github.actor != 'open-resource-broker-cicd[bot]'
97108
runs-on: ubuntu-latest
98109
needs: [config, setup-cache]
110+
# Override the workflow-level concurrency so this job is never cancelled
111+
# mid-run. If it were interrupted after `ruff` rewrites files but before
112+
# `git push`, the PR branch would be left in a half-formatted state.
113+
concurrency:
114+
group: ci-auto-format-${{ github.ref }}
115+
cancel-in-progress: false
99116
permissions:
100117
contents: write
101118
steps:
@@ -134,7 +151,7 @@ jobs:
134151
run: |
135152
git add .
136153
if ! git diff --staged --quiet; then
137-
git commit -m "style: auto-format code with ruff [skip ci]"
154+
git commit -m "style: auto-format code with ruff"
138155
git push
139156
fi
140157

‎.github/workflows/ci-tests.yml‎

Lines changed: 37 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,13 @@ on:
2222
schedule:
2323
- cron: '0 2 * * *'
2424

25+
# Cancel in-progress runs when a new push arrives on the same branch/ref.
26+
# Scheduled nightly runs use a stable group key so they never cancel each
27+
# other (there is only ever one running at a time by schedule).
28+
concurrency:
29+
group: ci-${{ github.workflow }}-${{ github.ref }}
30+
cancel-in-progress: true
31+
2532
permissions:
2633
contents: read
2734

@@ -84,12 +91,21 @@ jobs:
8491
environment: ${{ needs.config.outputs.environment }}
8592
testing-flag: ${{ needs.config.outputs.testing-flag }}
8693

87-
onmoto-tests:
88-
name: Onmoto Tests (Default Python)
94+
# Per-provider matrix. One job per provider subtree under
95+
# tests/providers/. New providers land here as new matrix entries —
96+
# no separate onmoto job needed (moto-mocked tests live under
97+
# tests/providers/aws/moto and run as part of the aws entry).
98+
providers-tests:
99+
name: Providers Tests
89100
needs: [config, setup-cache]
101+
strategy:
102+
fail-fast: false
103+
matrix:
104+
provider: [aws]
90105
uses: ./.github/workflows/reusable-test.yml
91106
with:
92-
test-type: onmoto
107+
test-type: providers
108+
provider: ${{ matrix.provider }}
93109
python-version: ${{ needs.config.outputs.default-python-version }}
94110
default-python-version: ${{ needs.config.outputs.default-python-version }}
95111
continue-on-error: false
@@ -99,12 +115,25 @@ jobs:
99115
environment: ${{ needs.config.outputs.environment }}
100116
testing-flag: ${{ needs.config.outputs.testing-flag }}
101117

102-
providers-tests:
103-
name: Providers & Infrastructure Tests (Default Python)
104-
needs: [config, setup-cache]
118+
# Serial-marked provider tests (currently the live-AWS subtree).
119+
# The make target points pytest directly at ``tests/providers/<provider>/live``
120+
# because ``norecursedirs`` excludes that path from auto-recursion. Without
121+
# ``--live`` the root conftest stamps every live test with ``skip_live``, so
122+
# the job collects the suite, reports SKIPPED, and exits 0 — that proves the
123+
# collection wiring + skip mechanism still work end-to-end on every PR.
124+
# When CI is wired up with real AWS credentials, set ``PYTEST_LIVE=--live``
125+
# in the workflow env to flip the suite into actual execution.
126+
providers-tests-serial:
127+
name: Providers Tests (Serial)
128+
needs: [config, setup-cache, providers-tests]
129+
strategy:
130+
fail-fast: false
131+
matrix:
132+
provider: [aws]
105133
uses: ./.github/workflows/reusable-test.yml
106134
with:
107-
test-type: providers
135+
test-type: providers-serial
136+
provider: ${{ matrix.provider }}
108137
python-version: ${{ needs.config.outputs.default-python-version }}
109138
default-python-version: ${{ needs.config.outputs.default-python-version }}
110139
continue-on-error: false
@@ -161,7 +190,7 @@ jobs:
161190
test-report:
162191
name: Generate Test Report
163192
runs-on: ubuntu-latest
164-
needs: [config, setup-cache, tests, integration-tests, e2e-tests, onmoto-tests, providers-tests, infrastructure-tests]
193+
needs: [config, setup-cache, tests, integration-tests, e2e-tests, providers-tests, providers-tests-serial, infrastructure-tests]
165194
if: always()
166195
permissions:
167196
contents: read

‎.github/workflows/reusable-test.yml‎

Lines changed: 31 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,14 @@ on:
44
workflow_call:
55
inputs:
66
test-type:
7-
description: 'Type of tests to run (unit, integration, e2e, performance)'
7+
description: 'Type of tests to run (unit, integration, e2e, performance, providers, infrastructure)'
88
required: true
99
type: string
10+
provider:
11+
description: 'When test-type=providers, scope the run to a single provider subtree (e.g. aws, k8s).'
12+
required: false
13+
type: string
14+
default: ''
1015
python-version:
1116
description: 'Python version to test with'
1217
required: false
@@ -65,7 +70,7 @@ env:
6570

6671
jobs:
6772
test:
68-
name: ${{ inputs.test-type == 'unit' && 'Unit Tests' || inputs.test-type == 'integration' && 'Integration Tests' || inputs.test-type == 'e2e' && 'End-to-End Tests' || format('{0} Tests', inputs.test-type) }}
73+
name: ${{ inputs.test-type == 'unit' && 'Unit Tests' || inputs.test-type == 'integration' && 'Integration Tests' || inputs.test-type == 'e2e' && 'End-to-End Tests' || (inputs.test-type == 'providers' && inputs.provider != '') && format('Providers Tests ({0})', inputs.provider) || format('{0} Tests', inputs.test-type) }}
6974
runs-on: ${{ inputs.os }}
7075
permissions:
7176
contents: read
@@ -81,14 +86,32 @@ jobs:
8186
fail-on-cache-miss: false
8287

8388
- name: Run tests
84-
run: make ci-tests-${{ inputs.test-type }}
89+
id: run-tests
90+
run: make ci-tests-${{ inputs.test-type }} PROVIDER=${{ inputs.provider }}
8591
continue-on-error: ${{ inputs.continue-on-error }}
8692

93+
# When continue-on-error is true the overall job status stays green even
94+
# if tests fail, which can hide regressions in the PR summary view.
95+
# This step fires only on failure and writes a visible warning to the
96+
# GitHub job summary so operators notice the failure without digging into
97+
# the raw log.
98+
- name: Annotate soft failure in job summary
99+
if: steps.run-tests.outcome == 'failure'
100+
run: |
101+
echo "::warning::${{ inputs.test-type }} tests failed (continue-on-error=true). Review the test log for details."
102+
{
103+
echo "## :warning: ${{ inputs.test-type }} test failures"
104+
echo ""
105+
echo "The **${{ inputs.test-type }}** test suite reported failures."
106+
echo "Overall job status is still green because \`continue-on-error: true\` is set,"
107+
echo "but these failures should be investigated before merging."
108+
} >> "$GITHUB_STEP_SUMMARY"
109+
87110
- name: Upload test results
88111
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
89112
if: always()
90113
with:
91-
name: test-results-${{ inputs.test-type }}-${{ inputs.os }}-py${{ inputs.python-version || inputs.default-python-version }}
114+
name: test-results-${{ inputs.test-type }}${{ inputs.provider != '' && format('-{0}', inputs.provider) || '' }}-${{ inputs.os }}-py${{ inputs.python-version || inputs.default-python-version }}
92115
retention-days: 30
93116
path: |
94117
junit-*.xml
@@ -100,14 +123,14 @@ jobs:
100123
uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v6
101124
with:
102125
token: ${{ secrets.CODECOV_TOKEN }}
103-
files: coverage-${{ inputs.test-type }}.xml
104-
flags: ${{ inputs.test-type }}
105-
name: ${{ inputs.test-type }}-tests
126+
files: coverage-${{ inputs.test-type }}${{ inputs.provider != '' && format('-{0}', inputs.provider) || '' }}.xml
127+
flags: ${{ inputs.test-type }}${{ inputs.provider != '' && format('-{0}', inputs.provider) || '' }}
128+
name: ${{ inputs.test-type }}${{ inputs.provider != '' && format('-{0}', inputs.provider) || '' }}-tests
106129

107130
- name: Upload test results to Codecov
108131
if: ${{ !cancelled() }}
109132
continue-on-error: true
110133
uses: codecov/test-results-action@0fa95f0e1eeaafde2c782583b36b28ad0d8c77d3 # v1
111134
with:
112135
token: ${{ secrets.CODECOV_TOKEN }}
113-
files: junit-${{ inputs.test-type }}.xml
136+
files: junit-${{ inputs.test-type }}${{ inputs.provider != '' && format('-{0}', inputs.provider) || '' }}.xml

‎.gitignore‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,7 @@
1+
.states
2+
.web
3+
reflex.lock/
4+
assets/external/
15
# Generated files
26
# pyproject.toml is now tracked - only metadata is templated, dependencies managed by Dependabot
37

@@ -279,3 +283,6 @@ coverage.json
279283

280284
# Backup files from editors/tools
281285
*.bak*
286+
287+
# Per-package coverage artifacts (never commit)
288+
coverage-*.xml

‎deployment/docker/docker-compose.prod.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ services:
3838
HF_LOGGING_LEVEL: INFO
3939
HF_LOGGING_CONSOLE_ENABLED: true
4040
HF_LOGGING_FILE_ENABLED: true
41-
HF_LOGGING_FILE_PATH: /app/logs/app.log
41+
HF_LOGGING_FILE_PATH: /app/logs/orb.log
4242

4343
# Storage configuration
4444
HF_STORAGE_STRATEGY: ${HF_STORAGE_STRATEGY:-json}

‎deployment/docker/docker-entrypoint.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -177,8 +177,8 @@ start_application() {
177177
cmd_args+=("--log-level" "${HF_LOGGING_LEVEL}")
178178
fi
179179

180-
# Add system serve command
181-
cmd_args+=("system" "serve")
180+
# Add server start command (PID 1 in the container — run foreground).
181+
cmd_args+=("server" "start" "--foreground")
182182

183183
# Add server-specific arguments
184184
if [[ -n "${HF_SERVER_HOST}" ]]; then

‎dev-tools/release/export_openapi_spec.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ else
3131
CONFIG_FLAG=(--config "$SPEC_TMP/config/config.json")
3232
fi
3333

34-
orb "${CONFIG_FLAG[@]}" system serve --socket-path "$SOCK" &
34+
orb "${CONFIG_FLAG[@]}" server start --foreground --api-only --socket-path "$SOCK" &
3535
SERVER_PID=$!
3636

3737
for _ in $(seq 1 30); do

‎makefiles/ci.mk‎

Lines changed: 55 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -78,22 +78,44 @@ ci-build-sbom: ## Generate SBOM files (matches publish.yml workflow)
7878
@echo "This matches the GitHub Actions publish.yml workflow exactly"
7979
$(MAKE) sbom-generate
8080

81+
# pytest-xdist parallelisation.
82+
#
83+
# Two variants are provided:
84+
#
85+
# PYTEST_PARALLEL_LOCAL — used by the local ``make test`` targets.
86+
# ``-n auto`` spawns one worker per CPU core, which is ideal on developer
87+
# machines that typically have 8-16 cores.
88+
#
89+
# PYTEST_PARALLEL_CI — used by CI targets (ci-tests-*).
90+
# GitHub-hosted runners expose exactly 2 vCPUs. ``-n auto`` therefore
91+
# spawns 2 workers, but the xdist scheduler introduces coordination
92+
# overhead that can make single-worker runs *slower* on 2-core hosts.
93+
# Using ``-n 2`` is explicit and avoids surprises if the runner class
94+
# changes. ``--dist=loadscope`` keeps tests in the same class/module on
95+
# one worker so class-scoped setUp / fixtures don't re-run per worker.
96+
#
97+
# PYTEST_PARALLEL (kept for backward compat) — points at the CI variant so
98+
# any external callers that reference the old variable name still work.
99+
#
100+
# Tests that share global state (live AWS, docker daemon, e2e tempdirs) are
101+
# tagged ``serial`` and run sequentially via a second pytest pass (PYTEST_SERIAL).
102+
PYTEST_PARALLEL_LOCAL := -n auto --dist=loadscope -m "not serial"
103+
PYTEST_PARALLEL_CI := -n 2 --dist=loadscope -m "not serial"
104+
PYTEST_PARALLEL := $(PYTEST_PARALLEL_CI)
105+
PYTEST_SERIAL := -m serial
106+
81107
ci-tests-unit: ## Run unit tests only (matches ci.yml unit-tests job)
82-
@echo "Running unit tests..."
83-
$(call run-tool,pytest,$(TESTS_UNIT) $(PYTEST_ARGS) $(PYTEST_COV_ARGS) --cov-report=xml:coverage-unit.xml --junitxml=junit-unit.xml)
108+
@echo "Running unit tests (parallel)..."
109+
$(call run-tool,pytest,$(TESTS_UNIT) $(PYTEST_PARALLEL) $(PYTEST_ARGS) $(PYTEST_COV_ARGS) --cov-report=xml:coverage-unit.xml --junitxml=junit-unit.xml)
84110

85111
ci-tests-integration: ## Run integration tests only (matches ci.yml integration-tests job)
86-
@echo "Running integration tests..."
87-
$(call run-tool,pytest,$(TESTS_INTEGRATION) $(PYTEST_ARGS) $(PYTEST_COV_ARGS) --cov-report=xml:coverage-integration.xml --junitxml=junit-integration.xml)
112+
@echo "Running integration tests (parallel)..."
113+
$(call run-tool,pytest,$(TESTS_INTEGRATION) $(PYTEST_PARALLEL) $(PYTEST_ARGS) $(PYTEST_COV_ARGS) --cov-report=xml:coverage-integration.xml --junitxml=junit-integration.xml)
88114

89115
ci-tests-e2e: ## Run end-to-end tests only (matches ci.yml e2e-tests job)
90116
@echo "Running end-to-end tests..."
91117
$(call run-tool,pytest,$(TESTS_E2E) $(PYTEST_ARGS) $(PYTEST_COV_ARGS) --cov-report=xml:coverage-e2e.xml --junitxml=junit-e2e.xml)
92118

93-
ci-tests-onmoto: ## Run onmoto (mocked AWS) tests only (matches ci.yml onmoto-tests job)
94-
@echo "Running onmoto tests..."
95-
$(call run-tool,pytest,$(TESTS_ONMOTO) $(PYTEST_ARGS) $(PYTEST_COV_ARGS) --cov-report=xml:coverage-onmoto.xml --junitxml=junit-onmoto.xml)
96-
97119
ci-tests-matrix: ## Run comprehensive test matrix (matches test-matrix.yml workflow)
98120
@echo "Running comprehensive test matrix..."
99121
$(call run-tool,pytest,$(TESTS) $(PYTEST_ARGS) $(PYTEST_COV_ARGS) --cov-report=xml:coverage-matrix.xml --junitxml=junit-matrix.xml)
@@ -102,13 +124,33 @@ ci-tests-performance: ## Run performance tests only (matches ci.yml performance
102124
@echo "Running performance tests..."
103125
$(call run-tool,pytest,$(TESTS_PERFORMANCE) $(PYTEST_ARGS) $(PYTEST_COV_ARGS) --cov-report=xml:coverage-performance.xml --junitxml=junit-performance.xml)
104126

105-
ci-tests-providers: ## Run providers tests only (matches ci.yml providers-tests job)
106-
@echo "Running providers tests..."
107-
$(call run-tool,pytest,$(TESTS_PROVIDERS) $(PYTEST_ARGS) $(PYTEST_COV_ARGS) --cov-report=xml:coverage-providers.xml --junitxml=junit-providers.xml)
127+
# Per-provider matrix target. Pass PROVIDER=<name> to scope the run to a
128+
# single provider's test subtree (e.g. PROVIDER=aws → tests/providers/aws).
129+
# CI's per-provider matrix invokes this with PROVIDER set for each entry;
130+
# local dev can omit it to run the full tests/providers tree.
131+
PROVIDER ?=
132+
PROVIDER_SUFFIX := $(if $(PROVIDER),-$(PROVIDER),)
133+
PROVIDER_PATH := $(if $(PROVIDER),$(TESTS_PROVIDERS)/$(PROVIDER),$(TESTS_PROVIDERS))
134+
# Serial-marked provider tests live under each provider's ``live/`` subtree.
135+
# That directory is listed in pyproject's ``norecursedirs`` so the parallel
136+
# ``ci-tests-providers`` target never descends into it; the serial target
137+
# below has to point pytest at the path explicitly so collection succeeds.
138+
# Without --live the root conftest adds ``skip_live`` to every collected
139+
# test, so the job exits 0 with a clear "163 skipped" line. CI with live
140+
# AWS credentials can opt in by setting PYTEST_LIVE=--live.
141+
PROVIDER_SERIAL_PATH := $(if $(PROVIDER),$(TESTS_PROVIDERS)/$(PROVIDER)/live,$(TESTS_PROVIDERS))
142+
PYTEST_LIVE ?=
143+
ci-tests-providers: ## Run providers tests (PROVIDER=<name> scopes to one provider)
144+
@echo "Running provider tests (parallel): $(if $(PROVIDER),$(PROVIDER),all)..."
145+
$(call run-tool,pytest,$(PROVIDER_PATH) $(PYTEST_PARALLEL) $(PYTEST_ARGS) $(PYTEST_COV_ARGS) --cov-report=xml:coverage-providers$(PROVIDER_SUFFIX).xml --junitxml=junit-providers$(PROVIDER_SUFFIX).xml)
146+
147+
ci-tests-providers-serial: ## Run the serial-marked subset of provider tests (live AWS, etc.)
148+
@echo "Running serial provider tests: $(if $(PROVIDER),$(PROVIDER),all)..."
149+
$(call run-tool,pytest,$(PROVIDER_SERIAL_PATH) $(PYTEST_SERIAL) $(PYTEST_LIVE) $(PYTEST_ARGS) $(PYTEST_COV_ARGS) --cov-report=xml:coverage-providers$(PROVIDER_SUFFIX)-serial.xml --junitxml=junit-providers$(PROVIDER_SUFFIX)-serial.xml)
108150

109151
ci-tests-infrastructure: ## Run infrastructure tests only (matches ci.yml infrastructure-tests job)
110-
@echo "Running infrastructure tests..."
111-
$(call run-tool,pytest,$(TESTS_INFRASTRUCTURE) $(PYTEST_ARGS) $(PYTEST_COV_ARGS) --cov-report=xml:coverage-infrastructure.xml --junitxml=junit-infrastructure.xml)
152+
@echo "Running infrastructure tests (parallel)..."
153+
$(call run-tool,pytest,$(TESTS_INFRASTRUCTURE) $(PYTEST_PARALLEL) $(PYTEST_ARGS) $(PYTEST_COV_ARGS) --cov-report=xml:coverage-infrastructure.xml --junitxml=junit-infrastructure.xml)
112154

113155
ci-check: ## Run comprehensive CI checks (matches GitHub Actions exactly)
114156
@echo "Running comprehensive CI checks that match GitHub Actions pipeline..."

‎makefiles/common.mk‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,6 @@ TESTS := tests
4242
TESTS_UNIT := $(TESTS)/unit
4343
TESTS_INTEGRATION := $(TESTS)/integration
4444
TESTS_E2E := $(TESTS)/e2e
45-
TESTS_ONMOTO := $(TESTS)/providers/aws/moto
4645
TESTS_PERFORMANCE := $(TESTS)/performance
4746
TESTS_INFRASTRUCTURE := $(TESTS)/infrastructure
4847
TESTS_PROVIDERS := $(TESTS)/providers

0 commit comments

Comments
 (0)