fix: detect partial provisioning and escalate to CRITICAL severity (SCT-501)

juliayakovlev · juliayakovlev · commit afed04d95232 · 2026-06-29T11:30:22.000+03:00
When pre-provisioning partially fails (e.g. GCE zone exhaustion after
creating some instances), the test setUp would silently continue with
fewer nodes than requested. This happened because add_nodes found the
partially-created instances via _get_instances but never validated
their count against the requested count.

Root cause: the GCE instance provider uses parallel API inserts, so
some instances can be created before a ZoneResourcesExhaustedError
terminates the batch. These orphaned instances persist in GCE with
matching TestId tags. When setUp later calls add_nodes, it discovers
them via _get_instances and proceeds without checking the count.

Additionally, provisioning failures raised during setUp were published
as TestFrameworkEvent with default ERROR severity, which does not
trigger the EventsAnalyzer interrupt mechanism (only CRITICAL does).

Changes:
- cluster_gce.py: validate len(instances) &gt;= count when pre-provisioned
  instances are found; raise ProvisionError if fewer than expected
- cluster_aws.py: add the same count validation on the non-REUSE
  pre-provisioned path (same bug pattern)
- tester.py: escalate ProvisionError and ProvisionUnrecoverableError
  to CRITICAL severity in teardown_on_exception so the EventsAnalyzer
  can interrupt the test run
- Add unit tests covering all three fixes
diff --git a/sdcm/cluster_aws.py b/sdcm/cluster_aws.py
@@ -43,6 +43,7 @@
 )
 from sdcm.provision.scylla_yaml import SeedProvider
 from sdcm.provision.helpers.cloud_init import wait_cloud_init_completes
+from sdcm.provision.provisioner import ProvisionError
 from sdcm.sct_provision.aws.cluster import PlacementGroup
 
 from sdcm.remote import LocalCmdRunner, shell_script_cmd, NETWORK_EXCEPTIONS
@@ -512,6 +513,11 @@ def _create_or_find_instances(
             self.log.info("Found instances to be reused from test [%s] = %s", self.test_config.REUSE_CLUSTER, instances)
             return instances
         if instances := self._get_instances(dc_idx, az_idx):
+            if len(instances) < count:
+                raise ProvisionError(
+                    f"Found only {len(instances)} pre-provisioned instance(s) but {count} were requested "
+                    f"for dc_idx={dc_idx}, az_idx={az_idx}. The pre-provisioning step may have partially failed."
+                )
             self.log.info("Found provisioned instances = %s", instances)
             return instances
         self.log.info("Found no provisioned instances. Provision them.")
diff --git a/sdcm/cluster_gce.py b/sdcm/cluster_gce.py
@@ -661,6 +661,11 @@ def add_nodes(
             if not instances:
                 raise RuntimeError("No nodes found for testId %s " % (self.test_config.test_id(),))
         elif instances := self._get_instances(instance_dc):
+            if len(instances) < count:
+                raise ProvisionError(
+                    f"Found only {len(instances)} pre-provisioned instance(s) but {count} were requested "
+                    f"for dc_idx={instance_dc}. The pre-provisioning step may have partially failed."
+                )
             self.log.info("Found provisioned instances = %s", instances)
         else:
             self.log.info("Found no provisioned instances. Provision them.")
diff --git a/sdcm/tester.py b/sdcm/tester.py
@@ -92,7 +92,7 @@
 from sdcm.provision.gce.provisioner import GceProvisioner
 from sdcm.provision.network_configuration import ssh_connection_ip_type
 from sdcm.provision.oci.provisioner import OciProvisioner
-from sdcm.provision.provisioner import provisioner_factory
+from sdcm.provision.provisioner import ProvisionError, ProvisionUnrecoverableError, provisioner_factory
 from sdcm.provision.helpers.certificate import (
     create_ca,
     update_certificate,
@@ -258,8 +258,19 @@ def wrapper(*args, **kwargs):
         try:
             return method(*args, **kwargs)
         except Exception as exc:
+            # Provisioning errors are always forced to CRITICAL to trigger
+            # EventsAnalyzer interrupts (exc.severity is NOT preserved for them).
+            # Non-provisioning exceptions forward exc.severity when present;
+            # otherwise leave as None so TestFrameworkEvent defaults to ERROR.
+            if isinstance(exc, (ProvisionError, ProvisionUnrecoverableError)):
+                severity = Severity.CRITICAL
+            else:
+                severity = getattr(exc, "severity", None)
             TestFrameworkEvent(
-                source=args[0].__class__.__name__, source_method="SetUp", exception=exc
+                source=args[0].__class__.__name__,
+                source_method="SetUp",
+                exception=exc,
+                severity=severity,
             ).publish_or_dump()
             TEST_LOG.exception("Exception in %s. Will call tearDown", method.__name__)
             # Try to initialize Argus if it hasn't been done yet, so the failure
diff --git a/unit_tests/unit/test_partial_provision_guard.py b/unit_tests/unit/test_partial_provision_guard.py
@@ -0,0 +1,260 @@
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See LICENSE for more details.
+#
+# Copyright (c) 2026 ScyllaDB
+
+"""Tests that partial provisioning is detected and raises errors.
+
+Covers SCT-501: when pre-provisioning partially fails (e.g. zone exhaustion
+after creating some instances), the test setUp should not silently continue
+with fewer nodes than requested.
+"""
+
+from unittest.mock import MagicMock, patch, PropertyMock
+
+import pytest
+
+from sdcm.provision.provisioner import ProvisionError, ProvisionUnrecoverableError, ZoneResourcesExhaustedError
+from sdcm.sct_events import Severity
+from sdcm.sct_events.system import TestFrameworkEvent
+from sdcm.tester import teardown_on_exception
+
+
+# ---------------------------------------------------------------------------
+# GCE: add_nodes count validation
+# ---------------------------------------------------------------------------
+
+
+def _make_gce_cluster(monkeypatch, instance_count, requested_count, is_reuse=False):
+    """Create a minimally-mocked GCECluster and invoke add_nodes.
+
+    Args:
+        monkeypatch: pytest monkeypatch fixture.
+        instance_count: how many pre-provisioned instances _get_instances returns.
+        requested_count: the count passed to add_nodes.
+        is_reuse: whether REUSE_CLUSTER is set.
+    """
+    from sdcm.cluster_gce import GCECluster
+
+    fake_instances = [MagicMock(name=f"instance-{i}") for i in range(instance_count)]
+
+    cluster = MagicMock(spec=GCECluster)
+    cluster.log = MagicMock()
+    cluster.params = MagicMock()
+    cluster.params.get.side_effect = lambda key, *a, **kw: {
+        "simulated_regions": False,
+    }.get(key)
+    cluster._node_index = 0
+    cluster.nodes = []
+    cluster.racks_count = 1
+
+    test_config = MagicMock()
+    test_config.REUSE_CLUSTER = is_reuse
+    test_config.test_id.return_value = "test-id-123"
+    cluster.test_config = test_config
+
+    cluster._get_instances = MagicMock(return_value=fake_instances)
+    cluster._create_node = MagicMock(side_effect=lambda inst, idx, dc, rack, after_config=None: MagicMock(name=f"node-{idx}"))
+
+    # Call the real add_nodes with self=cluster
+    return GCECluster.add_nodes.__wrapped__(
+        cluster, count=requested_count, dc_idx=0, rack=0,
+    )
+
+
+def test_gce_add_nodes_partial_provision_raises_error(monkeypatch):
+    """add_nodes should raise ProvisionError when fewer instances are found than requested."""
+    with pytest.raises(ProvisionError, match=r"Found only 2.*but 9 were requested"):
+        _make_gce_cluster(monkeypatch, instance_count=2, requested_count=9)
+
+
+def test_gce_add_nodes_exact_count_succeeds(monkeypatch):
+    """add_nodes should succeed when exactly the requested number of instances are found."""
+    result = _make_gce_cluster(monkeypatch, instance_count=3, requested_count=3)
+    assert len(result) == 3
+
+
+def test_gce_add_nodes_more_instances_than_requested_succeeds(monkeypatch):
+    """add_nodes should succeed when more instances are found than requested (extra instances are used)."""
+    result = _make_gce_cluster(monkeypatch, instance_count=5, requested_count=3)
+    assert len(result) == 5
+
+
+def test_gce_add_nodes_no_instances_provisions_inline(monkeypatch):
+    """add_nodes should fall through to inline provisioning when no instances are found."""
+    from sdcm.cluster_gce import GCECluster
+
+    cluster = MagicMock(spec=GCECluster)
+    cluster.log = MagicMock()
+    cluster.params = MagicMock()
+    cluster.params.get.side_effect = lambda key, *a, **kw: {
+        "simulated_regions": False,
+    }.get(key)
+    cluster._node_index = 0
+    cluster.nodes = []
+    cluster.racks_count = 1
+
+    test_config = MagicMock()
+    test_config.REUSE_CLUSTER = False
+    cluster.test_config = test_config
+
+    # _get_instances returns empty list -> falls through to _create_instances
+    cluster._get_instances = MagicMock(return_value=[])
+
+    fake_vms = [MagicMock(name=f"vm-{i}") for i in range(3)]
+    cluster._create_instances = MagicMock(return_value=fake_vms)
+    cluster._get_instance_with_retry = MagicMock(side_effect=lambda name, dc_idx: MagicMock(name=name))
+    cluster._create_node = MagicMock(side_effect=lambda inst, idx, dc, rack, after_config=None: MagicMock(name=f"node-{idx}"))
+
+    result = GCECluster.add_nodes.__wrapped__(cluster, count=3, dc_idx=0, rack=0)
+    cluster._create_instances.assert_called_once_with(3, 0, instance_type=None)
+    assert len(result) == 3
+
+
+# ---------------------------------------------------------------------------
+# AWS: _create_or_find_instances count validation
+# ---------------------------------------------------------------------------
+
+
+def test_aws_create_or_find_partial_provision_raises_error():
+    """AWS _create_or_find_instances should raise ProvisionError when fewer instances found than requested."""
+    from sdcm.cluster_aws import AWSCluster
+
+    cluster = MagicMock(spec=AWSCluster)
+    cluster.log = MagicMock()
+    cluster.params = MagicMock()
+    cluster.params.get.side_effect = lambda key, *a, **kw: {
+        "simulated_racks": False,
+    }.get(key)
+    cluster.nodes = []
+
+    test_config = MagicMock()
+    test_config.REUSE_CLUSTER = False
+    cluster.test_config = test_config
+
+    fake_instances = [MagicMock() for _ in range(2)]
+    cluster._get_instances = MagicMock(return_value=fake_instances)
+
+    with pytest.raises(ProvisionError, match=r"Found only 2.*but 5 were requested"):
+        AWSCluster._create_or_find_instances(cluster, count=5, ec2_user_data="", dc_idx=0, az_idx=0)
+
+
+def test_aws_create_or_find_exact_count_succeeds():
+    """AWS _create_or_find_instances should succeed when exactly the requested count is found."""
+    from sdcm.cluster_aws import AWSCluster
+
+    cluster = MagicMock(spec=AWSCluster)
+    cluster.log = MagicMock()
+    cluster.params = MagicMock()
+    cluster.params.get.side_effect = lambda key, *a, **kw: {
+        "simulated_racks": False,
+    }.get(key)
+    cluster.nodes = []
+
+    test_config = MagicMock()
+    test_config.REUSE_CLUSTER = False
+    cluster.test_config = test_config
+
+    fake_instances = [MagicMock() for _ in range(5)]
+    cluster._get_instances = MagicMock(return_value=fake_instances)
+
+    result = AWSCluster._create_or_find_instances(cluster, count=5, ec2_user_data="", dc_idx=0, az_idx=0)
+    assert len(result) == 5
+
+
+# ---------------------------------------------------------------------------
+# teardown_on_exception: severity escalation for provisioning errors
+# ---------------------------------------------------------------------------
+
+
+def _make_tester_mock():
+    """Create a minimal mock of ClusterTester for teardown_on_exception tests."""
+    tester = MagicMock()
+    tester.__class__.__name__ = "ClusterTester"
+    tester.params = MagicMock()
+    tester.argus_heartbeat_stop_signal = MagicMock()
+    tester.tearDown = MagicMock()
+    return tester
+
+
+def test_teardown_on_exception_provision_error_publishes_critical():
+    """ProvisionError in setUp should publish TestFrameworkEvent with CRITICAL severity."""
+    tester = _make_tester_mock()
+
+    @teardown_on_exception
+    def fake_setup(self):
+        raise ProvisionError("Failed to create instances")
+
+    published_events = []
+
+    with patch.object(TestFrameworkEvent, "publish_or_dump", side_effect=lambda: published_events.append(True)):
+        with patch("sdcm.tester.TestFrameworkEvent") as mock_event_cls:
+            mock_event_cls.return_value.publish_or_dump = MagicMock()
+            with pytest.raises(ProvisionError):
+                fake_setup(tester)
+            mock_event_cls.assert_called_once()
+            call_kwargs = mock_event_cls.call_args
+            assert call_kwargs.kwargs.get("severity") == Severity.CRITICAL or call_kwargs[1].get("severity") == Severity.CRITICAL
+
+
+def test_teardown_on_exception_zone_exhausted_publishes_critical():
+    """ZoneResourcesExhaustedError in setUp should publish TestFrameworkEvent with CRITICAL severity."""
+    tester = _make_tester_mock()
+
+    @teardown_on_exception
+    def fake_setup(self):
+        raise ZoneResourcesExhaustedError("Zone us-east1-d exhausted")
+
+    with patch("sdcm.tester.TestFrameworkEvent") as mock_event_cls:
+        mock_event_cls.return_value.publish_or_dump = MagicMock()
+        with pytest.raises(ZoneResourcesExhaustedError):
+            fake_setup(tester)
+        mock_event_cls.assert_called_once()
+        call_kwargs = mock_event_cls.call_args
+        assert call_kwargs.kwargs.get("severity") == Severity.CRITICAL or call_kwargs[1].get("severity") == Severity.CRITICAL
+
+
+def test_teardown_on_exception_generic_error_publishes_default_severity():
+    """Non-provisioning errors in setUp should publish TestFrameworkEvent with default (None) severity."""
+    tester = _make_tester_mock()
+
+    @teardown_on_exception
+    def fake_setup(self):
+        raise RuntimeError("Something else went wrong")
+
+    with patch("sdcm.tester.TestFrameworkEvent") as mock_event_cls:
+        mock_event_cls.return_value.publish_or_dump = MagicMock()
+        with pytest.raises(RuntimeError):
+            fake_setup(tester)
+        mock_event_cls.assert_called_once()
+        call_kwargs = mock_event_cls.call_args
+        assert call_kwargs.kwargs.get("severity") is None or call_kwargs[1].get("severity") is None
+
+
+def test_teardown_on_exception_preserves_severity_from_exception():
+    """If the exception carries a severity attribute, it should be forwarded to the event."""
+    tester = _make_tester_mock()
+
+    class ErrorWithSeverity(Exception):
+        severity = Severity.WARNING
+
+    @teardown_on_exception
+    def fake_setup(self):
+        raise ErrorWithSeverity("Warning-level error")
+
+    with patch("sdcm.tester.TestFrameworkEvent") as mock_event_cls:
+        mock_event_cls.return_value.publish_or_dump = MagicMock()
+        with pytest.raises(ErrorWithSeverity):
+            fake_setup(tester)
+        mock_event_cls.assert_called_once()
+        call_kwargs = mock_event_cls.call_args
+        severity = call_kwargs.kwargs.get("severity") or call_kwargs[1].get("severity")
+        assert severity == Severity.WARNING
