test(live): assert capacity-unit fulfilment + template-aware count

Replace the relaxed `len(machine_ids) >= 1` assertions in the four live
test files with a two-part check:

1. Capacity-unit fulfilment: assert fulfilled_units >= target_units using
   the new fields from the provider response
   (status_response["requests"][0]["target_units"] /
   ["fulfilled_units"]).  When the fields are absent (older build) falls
   back to capacity and len(machine_ids) so existing runs are unaffected.

2. Template-aware instance count: if template_uses_weighted_capacity()
   returns True, assert at least one instance was launched (weighted
   capacity units can fulfil with fewer physical instances); otherwise
   assert exactly `capacity` instances were provisioned.

Files changed: test_sdk_onaws.py, test_mcp_onaws.py,
test_cleanup_e2e_onaws.py, test_onaws.py (provide_release_control_loop).

Author: fgogolli

tests/providers/aws/live/test_cleanup_e2e_onaws.py

@@ -467,14 +467,26 @@ async def _run_cleanup_verification(
 
     # 3. Assert instances provisioned
     machine_ids = _extract_machine_ids(status_response)
-    # Weighted templates can fulfill capacity with fewer instances (AWS
-    # WeightedCapacity). The COMPLETED status above already proves the fleet
-    # reached its target capacity; assert only that at least one instance was
-    # launched.
-    assert len(machine_ids) >= 1, (
-        f"Expected at least one machine after complete status, got: {machine_ids}"
+
+    # Capacity-unit fulfilment check from provider response fields.
+    # target_units / fulfilled_units reflect capacity units for weighted fleets
+    # and instance count for unweighted templates (1 unit == 1 instance).
+    _req0 = status_response.get("requests", [{}])[0] if isinstance(status_response, dict) else {}
+    target_units = _req0.get("target_units") or capacity
+    fulfilled_units = _req0.get("fulfilled_units") or len(machine_ids)
+    assert fulfilled_units >= target_units, (
+        f"Fleet not fully fulfilled: fulfilled={fulfilled_units}, target={target_units}"
     )
-    _ = capacity  # retained for log context
+
+    # Template-aware instance count sanity check.
+    if scenarios.template_uses_weighted_capacity(test_case):
+        assert len(machine_ids) >= 1, (
+            f"Expected at least 1 machine (weighted template), got: {machine_ids}"
+        )
+    else:
+        assert len(machine_ids) == capacity, (
+            f"Expected {capacity} machines (unweighted template), got {len(machine_ids)}: {machine_ids}"
+        )
 
     returned_id = status_response.get("requests", [{}])[0].get("request_id") or status_response.get(
         "requests", [{}]

tests/providers/aws/live/test_mcp_onaws.py

@@ -343,16 +343,27 @@ async def _run_full_cycle_mcp(mcp_server, test_case: dict, tracked_request_ids:
     )
 
     machine_ids = _extract_machine_ids(status_result)
-    # Weighted templates (vmTypes with weight > 1) launch fewer instances than the
-    # requested capacity units (AWS WeightedCapacity semantics). Assert that the
-    # fleet was fulfilled (status == complete already proves this) and that at
-    # least one instance is healthy. For unweighted templates instance count
-    # equals capacity, but the strict equality is enforced by the provider via
-    # the COMPLETED gate, not by this assertion.
-    assert len(machine_ids) >= 1, (
-        f"Expected at least one machine after complete status, got: {machine_ids}"
+
+    # Capacity-unit fulfilment check from provider response fields.
+    # target_units / fulfilled_units reflect capacity units for weighted fleets
+    # and instance count for unweighted templates (1 unit == 1 instance).
+    _req0 = status_result.get("requests", [{}])[0] if isinstance(status_result, dict) else {}
+    target_units = _req0.get("target_units") or capacity
+    fulfilled_units = _req0.get("fulfilled_units") or len(machine_ids)
+    assert fulfilled_units >= target_units, (
+        f"Fleet not fully fulfilled: fulfilled={fulfilled_units}, target={target_units}"
     )
 
+    # Template-aware instance count sanity check.
+    if scenarios.template_uses_weighted_capacity(test_case):
+        assert len(machine_ids) >= 1, (
+            f"Expected at least 1 machine (weighted template), got: {machine_ids}"
+        )
+    else:
+        assert len(machine_ids) == capacity, (
+            f"Expected {capacity} machines (unweighted template), got {len(machine_ids)}: {machine_ids}"
+        )
+
     for machine_id in machine_ids:
         state = get_instance_state(machine_id)
         assert state["exists"], f"Instance {machine_id} not found in AWS"

tests/providers/aws/live/test_onaws.py

@@ -1482,6 +1482,31 @@ def provide_release_control_loop(hfm, template_json, capacity_to_request, test_c
 
     _check_all_ec2_hosts_are_being_provisioned(status_response)
 
+    # Capacity-unit fulfilment check from provider response fields.
+    # target_units / fulfilled_units reflect capacity units for weighted fleets
+    # and instance count for unweighted templates (1 unit == 1 instance).
+    _req0 = status_response.get("requests", [{}])[0] if isinstance(status_response, dict) else {}
+    _machine_ids_for_check = [
+        m.get("machineId") or m.get("machine_id")
+        for m in _req0.get("machines", [])
+        if m.get("machineId") or m.get("machine_id")
+    ]
+    _target_units = _req0.get("target_units") or capacity_to_request
+    _fulfilled_units = _req0.get("fulfilled_units") or len(_machine_ids_for_check)
+    assert _fulfilled_units >= _target_units, (
+        f"Fleet not fully fulfilled: fulfilled={_fulfilled_units}, target={_target_units}"
+    )
+    # Template-aware instance count sanity check.
+    if test_case and scenarios.template_uses_weighted_capacity(test_case):
+        assert len(_machine_ids_for_check) >= 1, (
+            f"Expected at least 1 machine (weighted template), got: {_machine_ids_for_check}"
+        )
+    else:
+        assert len(_machine_ids_for_check) == capacity_to_request, (
+            f"Expected {capacity_to_request} machines (unweighted template), "
+            f"got {len(_machine_ids_for_check)}: {_machine_ids_for_check}"
+        )
+
     # Validate instance attributes against template
     log.info("Starting instance attribute validation against template")
     attribute_validation_passed = validate_random_instance_attributes(

tests/providers/aws/live/test_sdk_onaws.py

@@ -310,16 +310,27 @@ async def _run_full_cycle(sdk, test_case: dict, tracked_request_ids: list[str])
     )
 
     machine_ids = _extract_machine_ids(status_response)
-    # Weighted templates (vmTypes with weight > 1) launch fewer instances than the
-    # requested capacity units (AWS WeightedCapacity semantics). Assert that the
-    # fleet was fulfilled (status == complete already proves this) and that at
-    # least one instance is healthy. For unweighted templates instance count
-    # equals capacity, but the strict equality is enforced by the provider via
-    # the COMPLETED gate, not by this assertion.
-    assert len(machine_ids) >= 1, (
-        f"Expected at least one machine after complete status, got: {machine_ids}"
+
+    # Capacity-unit fulfilment check from provider response fields.
+    # target_units / fulfilled_units reflect capacity units for weighted fleets
+    # and instance count for unweighted templates (1 unit == 1 instance).
+    _req0 = status_response.get("requests", [{}])[0] if isinstance(status_response, dict) else {}
+    target_units = _req0.get("target_units") or capacity
+    fulfilled_units = _req0.get("fulfilled_units") or len(machine_ids)
+    assert fulfilled_units >= target_units, (
+        f"Fleet not fully fulfilled: fulfilled={fulfilled_units}, target={target_units}"
     )
 
+    # Template-aware instance count sanity check.
+    if scenarios.template_uses_weighted_capacity(test_case):
+        assert len(machine_ids) >= 1, (
+            f"Expected at least 1 machine (weighted template), got: {machine_ids}"
+        )
+    else:
+        assert len(machine_ids) == capacity, (
+            f"Expected {capacity} machines (unweighted template), got {len(machine_ids)}: {machine_ids}"
+        )
+
     for machine_id in machine_ids:
         state = get_instance_state(machine_id)
         assert state["exists"], f"Instance {machine_id} not found in AWS"