Skip to content

Commit 590230c

Browse files
fgogollifgogolli
authored
Merge pull request #190 from awslabs/test/onmoto-full-aws-coverage
fix(providers): request fleet partial return regression, Rich JSON output, onmoto coverage + pyright CI
2 parents ba0042b + ef80687 commit 590230c

130 files changed

Lines changed: 5812 additions & 734 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/ci-tests.yml‎

Lines changed: 77 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,81 @@ jobs:
5454
environment: ${{ needs.config.outputs.environment }}
5555
testing-flag: ${{ needs.config.outputs.testing-flag }}
5656

57+
integration-tests:
58+
name: Integration Tests (Default Python)
59+
needs: [config, setup-cache]
60+
uses: ./.github/workflows/reusable-test.yml
61+
with:
62+
test-type: integration
63+
python-version: ${{ needs.config.outputs.default-python-version }}
64+
default-python-version: ${{ needs.config.outputs.default-python-version }}
65+
continue-on-error: false
66+
aws-region: ${{ needs.config.outputs.aws-region }}
67+
aws-access-key: ${{ needs.config.outputs.aws-access-key }}
68+
aws-secret-key: ${{ needs.config.outputs.aws-secret-key }}
69+
environment: ${{ needs.config.outputs.environment }}
70+
testing-flag: ${{ needs.config.outputs.testing-flag }}
71+
72+
e2e-tests:
73+
name: End-to-End Tests (Default Python)
74+
needs: [config, setup-cache]
75+
uses: ./.github/workflows/reusable-test.yml
76+
with:
77+
test-type: e2e
78+
python-version: ${{ needs.config.outputs.default-python-version }}
79+
default-python-version: ${{ needs.config.outputs.default-python-version }}
80+
continue-on-error: true
81+
aws-region: ${{ needs.config.outputs.aws-region }}
82+
aws-access-key: ${{ needs.config.outputs.aws-access-key }}
83+
aws-secret-key: ${{ needs.config.outputs.aws-secret-key }}
84+
environment: ${{ needs.config.outputs.environment }}
85+
testing-flag: ${{ needs.config.outputs.testing-flag }}
86+
87+
onmoto-tests:
88+
name: Onmoto Tests (Default Python)
89+
needs: [config, setup-cache]
90+
uses: ./.github/workflows/reusable-test.yml
91+
with:
92+
test-type: onmoto
93+
python-version: ${{ needs.config.outputs.default-python-version }}
94+
default-python-version: ${{ needs.config.outputs.default-python-version }}
95+
continue-on-error: true
96+
aws-region: ${{ needs.config.outputs.aws-region }}
97+
aws-access-key: ${{ needs.config.outputs.aws-access-key }}
98+
aws-secret-key: ${{ needs.config.outputs.aws-secret-key }}
99+
environment: ${{ needs.config.outputs.environment }}
100+
testing-flag: ${{ needs.config.outputs.testing-flag }}
101+
102+
providers-tests:
103+
name: Providers & Infrastructure Tests (Default Python)
104+
needs: [config, setup-cache]
105+
uses: ./.github/workflows/reusable-test.yml
106+
with:
107+
test-type: providers
108+
python-version: ${{ needs.config.outputs.default-python-version }}
109+
default-python-version: ${{ needs.config.outputs.default-python-version }}
110+
continue-on-error: false
111+
aws-region: ${{ needs.config.outputs.aws-region }}
112+
aws-access-key: ${{ needs.config.outputs.aws-access-key }}
113+
aws-secret-key: ${{ needs.config.outputs.aws-secret-key }}
114+
environment: ${{ needs.config.outputs.environment }}
115+
testing-flag: ${{ needs.config.outputs.testing-flag }}
116+
117+
infrastructure-tests:
118+
name: Infrastructure Tests (Default Python)
119+
needs: [config, setup-cache]
120+
uses: ./.github/workflows/reusable-test.yml
121+
with:
122+
test-type: infrastructure
123+
python-version: ${{ needs.config.outputs.default-python-version }}
124+
default-python-version: ${{ needs.config.outputs.default-python-version }}
125+
continue-on-error: false
126+
aws-region: ${{ needs.config.outputs.aws-region }}
127+
aws-access-key: ${{ needs.config.outputs.aws-access-key }}
128+
aws-secret-key: ${{ needs.config.outputs.aws-secret-key }}
129+
environment: ${{ needs.config.outputs.environment }}
130+
testing-flag: ${{ needs.config.outputs.testing-flag }}
131+
57132
build-and-package:
58133
name: Build & Package
59134
runs-on: ubuntu-latest
@@ -86,7 +161,7 @@ jobs:
86161
test-report:
87162
name: Generate Test Report
88163
runs-on: ubuntu-latest
89-
needs: [config, setup-cache, tests]
164+
needs: [config, setup-cache, tests, integration-tests, e2e-tests, onmoto-tests, providers-tests, infrastructure-tests]
90165
if: always()
91166
permissions:
92167
contents: read
@@ -114,7 +189,7 @@ jobs:
114189
if: always()
115190
continue-on-error: true
116191
with:
117-
files: "test-results/**/*.xml"
192+
files: "test-results/**/junit-*.xml"
118193
check_name: "Test Results Summary"
119194
comment_mode: "always"
120195
job_summary: true

‎.github/workflows/test-matrix.yml‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,6 @@ jobs:
4242
python-version: ${{ matrix.python-version }}
4343
os: ${{ matrix.os }}
4444
default-python-version: ${{ needs.config.outputs.default-python-version }}
45-
continue-on-error: true # TODO: Remove once tests are stable
4645
aws-region: ${{ needs.config.outputs.aws-region }}
4746
aws-access-key: ${{ needs.config.outputs.aws-access-key }}
4847
aws-secret-key: ${{ needs.config.outputs.aws-secret-key }}
@@ -51,7 +50,7 @@ jobs:
5150

5251
integration-tests:
5352
name: Integration Tests
54-
needs: [config, unit-tests]
53+
needs: [config]
5554
permissions:
5655
contents: read
5756
strategy:
@@ -67,7 +66,6 @@ jobs:
6766
python-version: ${{ matrix.python-version }}
6867
os: ${{ matrix.os }}
6968
default-python-version: ${{ needs.config.outputs.default-python-version }}
70-
continue-on-error: true # TODO: Remove once tests are stable
7169
aws-region: ${{ needs.config.outputs.aws-region }}
7270
aws-access-key: ${{ needs.config.outputs.aws-access-key }}
7371
aws-secret-key: ${{ needs.config.outputs.aws-secret-key }}
@@ -144,7 +142,7 @@ jobs:
144142
145143
e2e-tests:
146144
name: End-to-End Tests
147-
needs: [config, unit-tests, integration-tests]
145+
needs: [config]
148146
permissions:
149147
contents: read
150148
strategy:
@@ -160,7 +158,6 @@ jobs:
160158
python-version: ${{ matrix.python-version }}
161159
os: ${{ matrix.os }}
162160
default-python-version: ${{ needs.config.outputs.default-python-version }}
163-
continue-on-error: true # TODO: Remove once tests are stable
164161
aws-region: ${{ needs.config.outputs.aws-region }}
165162
aws-access-key: ${{ needs.config.outputs.aws-access-key }}
166163
aws-secret-key: ${{ needs.config.outputs.aws-secret-key }}

‎config/aws_flamurg-testing-Admin_eu-west-2_templates.json‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
{
2+
"scheduler_type": "hostfactory",
3+
"templates": [
4+
{
5+
"templateId": "tpl-new-001",
6+
"maxNumber": 1,
7+
"imageId": "ami-12345678",
8+
"priceType": "ondemand",
9+
"name": "new-template",
10+
"providerApi": "ec2_fleet",
11+
"is_active": true,
12+
"subnetId": "subnet-abc"
13+
},
14+
{
15+
"templateId": "tpl-flow-001",
16+
"maxNumber": 1,
17+
"imageId": "ami-abc",
18+
"priceType": "ondemand",
19+
"name": "tpl-flow-001",
20+
"providerApi": "ec2_fleet",
21+
"is_active": true
22+
},
23+
{
24+
"templateId": "tpl-create-use-001",
25+
"maxNumber": 1,
26+
"imageId": "ami-12345678",
27+
"priceType": "ondemand",
28+
"name": "lifecycle-template",
29+
"providerApi": "ec2_fleet",
30+
"is_active": true,
31+
"subnetId": "subnet-abc"
32+
},
33+
{
34+
"templateId": "tpl-no-bus",
35+
"maxNumber": 1,
36+
"imageId": "ami-abc",
37+
"priceType": "ondemand",
38+
"name": "tpl-no-bus",
39+
"providerApi": "ec2_fleet",
40+
"is_active": true
41+
}
42+
]
43+
}

‎config/default_config.json‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,15 @@
44
"type": "default"
55
},
66
"provider": {
7-
"providers": [],
7+
"providers": [
8+
{
9+
"name": "aws-default",
10+
"type": "aws",
11+
"enabled": true,
12+
"weight": 100,
13+
"config": {}
14+
}
15+
],
816
"selection_policy": "FIRST_AVAILABLE",
917
"default_provider_type": "aws",
1018
"health_check_interval": 60,

‎makefiles/ci.mk‎

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ ci-quality-radon: ## Run radon complexity analysis
1818

1919
ci-quality-pyright: ## Run pyright type checking
2020
@echo "Running pyright type check..."
21-
$(call run-tool,pyright,src/)
21+
$(call run-tool,pyright,)
2222

2323
# Composite target (for local convenience)
2424
ci-quality: ci-quality-ruff ci-quality-pyright ## Run all enforced code quality checks
@@ -84,19 +84,31 @@ ci-tests-unit: ## Run unit tests only (matches ci.yml unit-tests job)
8484

8585
ci-tests-integration: ## Run integration tests only (matches ci.yml integration-tests job)
8686
@echo "Running integration tests..."
87-
$(call run-tool,pytest,$(TESTS_INTEGRATION) $(PYTEST_ARGS) --junitxml=junit-integration.xml)
87+
$(call run-tool,pytest,$(TESTS_INTEGRATION) $(PYTEST_ARGS) $(PYTEST_COV_ARGS) --cov-report=xml:coverage-integration.xml --junitxml=junit-integration.xml)
8888

8989
ci-tests-e2e: ## Run end-to-end tests only (matches ci.yml e2e-tests job)
9090
@echo "Running end-to-end tests..."
91-
$(call run-tool,pytest,$(TESTS_E2E) $(PYTEST_ARGS) --junitxml=junit-e2e.xml)
91+
$(call run-tool,pytest,$(TESTS_E2E) $(PYTEST_ARGS) $(PYTEST_COV_ARGS) --cov-report=xml:coverage-e2e.xml --junitxml=junit-e2e.xml)
92+
93+
ci-tests-onmoto: ## Run onmoto (mocked AWS) tests only (matches ci.yml onmoto-tests job)
94+
@echo "Running onmoto tests..."
95+
$(call run-tool,pytest,$(TESTS_ONMOTO) $(PYTEST_ARGS) $(PYTEST_COV_ARGS) --cov-report=xml:coverage-onmoto.xml --junitxml=junit-onmoto.xml)
9296

9397
ci-tests-matrix: ## Run comprehensive test matrix (matches test-matrix.yml workflow)
9498
@echo "Running comprehensive test matrix..."
9599
$(call run-tool,pytest,$(TESTS) $(PYTEST_ARGS) $(PYTEST_COV_ARGS) --cov-report=xml:coverage-matrix.xml --junitxml=junit-matrix.xml)
96100

97101
ci-tests-performance: ## Run performance tests only (matches ci.yml performance-tests job)
98102
@echo "Running performance tests..."
99-
$(call run-tool,pytest,$(TESTS_PERFORMANCE) $(PYTEST_ARGS) --junitxml=junit-performance.xml)
103+
$(call run-tool,pytest,$(TESTS_PERFORMANCE) $(PYTEST_ARGS) $(PYTEST_COV_ARGS) --cov-report=xml:coverage-performance.xml --junitxml=junit-performance.xml)
104+
105+
ci-tests-providers: ## Run providers tests only (matches ci.yml providers-tests job)
106+
@echo "Running providers tests..."
107+
$(call run-tool,pytest,$(TESTS_PROVIDERS) $(PYTEST_ARGS) $(PYTEST_COV_ARGS) --cov-report=xml:coverage-providers.xml --junitxml=junit-providers.xml)
108+
109+
ci-tests-infrastructure: ## Run infrastructure tests only (matches ci.yml infrastructure-tests job)
110+
@echo "Running infrastructure tests..."
111+
$(call run-tool,pytest,$(TESTS_INFRASTRUCTURE) $(PYTEST_ARGS) $(PYTEST_COV_ARGS) --cov-report=xml:coverage-infrastructure.xml --junitxml=junit-infrastructure.xml)
100112

101113
ci-check: ## Run comprehensive CI checks (matches GitHub Actions exactly)
102114
@echo "Running comprehensive CI checks that match GitHub Actions pipeline..."

‎makefiles/common.mk‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,10 @@ TESTS := tests
4242
TESTS_UNIT := $(TESTS)/unit
4343
TESTS_INTEGRATION := $(TESTS)/integration
4444
TESTS_E2E := $(TESTS)/e2e
45+
TESTS_ONMOTO := $(TESTS)/onmoto
4546
TESTS_PERFORMANCE := $(TESTS)/performance
47+
TESTS_INFRASTRUCTURE := $(TESTS)/infrastructure
48+
TESTS_PROVIDERS := $(TESTS)/providers
4649
CONFIG := config/config.json
4750

4851
# Coverage settings
@@ -51,7 +54,7 @@ COVERAGE_HTML := htmlcov
5154

5255
# Test settings
5356
PYTEST_ARGS := -v --tb=short --durations=10
54-
PYTEST_COV_ARGS := --cov=$(PACKAGE_ROOT) --cov-report=term-missing --cov-branch --no-cov-on-fail
57+
PYTEST_COV_ARGS := --cov=$(PACKAGE_ROOT) --cov-report=term-missing --cov-branch
5558
PYTEST_TIMEOUT := --timeout=300
5659
PYTEST_MAXFAIL := --maxfail=5
5760

‎pyproject.toml‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -460,14 +460,14 @@ combine-as-imports = true
460460
[tool.pyright]
461461
pythonVersion = "3.12"
462462
pythonPlatform = "Linux"
463-
include = ["src/orb", "tests/shared", "tests/onmoto"] # keep in sync with .project.yml build.package_root
463+
include = ["src/orb", "tests"]
464464
exclude = [
465465
".venv", "build", "dist",
466-
# onmoto files with pre-existing type errors, excluded until resolved
467-
"tests/onmoto/test_config_driven_provision.py",
468-
"tests/onmoto/test_cqrs_control_loop.py",
469-
"tests/onmoto/test_hf_contract.py",
470-
"tests/onmoto/test_provision_lifecycle.py",
466+
# These tests intentionally import non-existent modules to verify deleted code is gone
467+
"tests/unit/test_asg_stack_deleted.py",
468+
"tests/unit/application/test_application_comprehensive.py",
469+
"tests/unit/infrastructure/test_infrastructure_comprehensive.py",
470+
"tests/unit/providers/test_providers_comprehensive.py",
471471
]
472472
extraPaths = ["."]
473473
typeCheckingMode = "basic"

‎src/orb/api/dependencies.py‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
from __future__ import annotations
44

5-
from typing import TypeVar
5+
from typing import Any, TypeVar
66

77
from orb.application.ports.scheduler_port import SchedulerPort
88
from orb.application.services.orchestration.acquire_machines import AcquireMachinesOrchestrator
@@ -151,3 +151,10 @@ def get_refresh_templates_orchestrator() -> RefreshTemplatesOrchestrator:
151151
def get_response_formatting_service() -> ResponseFormattingService:
152152
"""Get ResponseFormattingService from DI container."""
153153
return get_di_container().get(ResponseFormattingService)
154+
155+
156+
def get_health_check_port() -> Any:
157+
"""Get HealthCheckPort from DI container."""
158+
from orb.domain.base.ports.health_check_port import HealthCheckPort
159+
160+
return get_di_container().get(HealthCheckPort)

‎src/orb/api/server.py‎

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -4,22 +4,23 @@
44
from typing import TYPE_CHECKING, Any, cast
55

66
try:
7-
from fastapi import FastAPI
7+
from fastapi import Depends, FastAPI
88
from fastapi.middleware.cors import CORSMiddleware
99
from fastapi.middleware.trustedhost import TrustedHostMiddleware
1010
from fastapi.responses import JSONResponse, Response
1111

1212
FASTAPI_AVAILABLE = True
1313
except ImportError:
1414
FASTAPI_AVAILABLE = False
15+
Depends = None # type: ignore[assignment,misc]
1516
FastAPI = None # type: ignore[assignment,misc]
1617
CORSMiddleware = None # type: ignore[assignment,misc]
1718
TrustedHostMiddleware = None # type: ignore[assignment,misc]
1819
JSONResponse = None # type: ignore[assignment,misc]
1920
Response = None # type: ignore[assignment,misc]
2021

2122
if TYPE_CHECKING:
22-
from fastapi import FastAPI
23+
from fastapi import Depends, FastAPI
2324
from fastapi.middleware.cors import CORSMiddleware
2425
from fastapi.middleware.trustedhost import TrustedHostMiddleware
2526
from fastapi.responses import JSONResponse, Response
@@ -162,14 +163,12 @@ async def global_exception_handler(request: Any, exc: Exception) -> Any:
162163
)
163164

164165
# Add health check endpoint
166+
from orb.api.dependencies import get_health_check_port
167+
165168
@app.get("/health", tags=["System"])
166-
async def health_check() -> Any:
169+
async def health_check(health_port: Any = Depends(get_health_check_port)) -> Any: # type: ignore[misc]
167170
"""Health check endpoint."""
168-
from orb.domain.base.ports.health_check_port import HealthCheckPort
169-
from orb.infrastructure.di.container import get_container
170-
171171
try:
172-
health_port = get_container().get(HealthCheckPort)
173172
health_port.run_all_checks()
174173
status = health_port.get_status()
175174
except Exception:

‎src/orb/cli/console.py‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,14 @@
1111
from rich.console import Console
1212

1313
RICH_AVAILABLE = True
14-
_console = Console()
15-
_error_console = Console(stderr=True)
14+
_console = Console(
15+
no_color=not sys.stdout.isatty(), width=None if sys.stdout.isatty() else 2**31 - 1
16+
)
17+
_error_console = Console(
18+
stderr=True,
19+
no_color=not sys.stderr.isatty(),
20+
width=None if sys.stderr.isatty() else 2**31 - 1,
21+
)
1622
except ImportError:
1723
RICH_AVAILABLE = False
1824

0 commit comments

Comments
 (0)