feat: Supports interrupting the pre-release phase (#2876)

Author: xogas

apiserver/paasng/paas_wl/bk_app/cnative/specs/constants.py

@@ -28,6 +28,16 @@
 TENANT_GUARD_ANNO_KEY = "bkapp.paas.bk.tencent.com/tenant-guard"
 # workloads 注入到 annotations 的部署ID字段
 BKPAAS_DEPLOY_ID_ANNO_KEY = "bkapp.paas.bk.tencent.com/bkpaas-deploy-id"
+
+# 标记 "最近一次被平台请求中断的部署" 的注解键名, 主要用于中断 operator 的部署
+#
+# 取值: 被请求中断的那次部署的 deploy_id (字符串), 与 BKPAAS_DEPLOY_ID_ANNO_KEY 同源
+# 写入方: apiserver 在用户主动中断部署, 把当前部署的 deploy_id patch 到此注解,
+#        用于向 operator 发出实时中断信号
+# 读取方: operator 在调和 BkApp 时, 仅当本注解的值与 BKPAAS_DEPLOY_ID_ANNO_KEY 的值同时存在且相等时,
+#        认为"当前正在进行的这次部署已被用户主动中断"
+INTERRUPTED_DEPLOY_ID_ANNO_KEY = "bkapp.paas.bk.tencent.com/interrupted-deploy-id"
+
 # workloads 注入到 annotations 的增强服务信息字段
 BKPAAS_ADDONS_ANNO_KEY = "bkapp.paas.bk.tencent.com/addons"
 # 注解中存储 region 的键名

apiserver/paasng/paasng/platform/engine/deploy/bg_command/bkapp_hook.py

@@ -92,8 +92,13 @@ def _perform(self, hook_name: str) -> PodPhase:
             return PodPhase.RUNNING
 
     def _mark_step_stop(self, status: PodPhase):
+        self.deployment.refresh_from_db(fields=["release_int_requested_at"])
+        is_interrupted = status != PodPhase.SUCCEEDED and self.deployment.has_requested_int
+
         if status == PodPhase.SUCCEEDED:
             self.stream.write_message(Style.Warning("Pre-release execution succeed"))
+        elif is_interrupted:
+            self.stream.write_message(Style.Warning("Pre-release interrupted"))
         elif status == PodPhase.FAILED:
             self.stream.write_message(Style.Error("Pre-release failed, please check logs for more details"))
         else:
@@ -106,5 +111,9 @@ def _mark_step_stop(self, status: PodPhase):
 
         if status == PodPhase.SUCCEEDED:
             step.mark_and_write_to_stream(self.stream, JobStatus.SUCCESSFUL)
-        elif status == PodPhase.FAILED:
+            return
+        if is_interrupted:
+            step.mark_and_write_to_stream(self.stream, JobStatus.INTERRUPTED, {"message": "Pre-release interrupted"})
+            return
+        if status == PodPhase.FAILED:
             step.mark_and_write_to_stream(self.stream, JobStatus.FAILED)

apiserver/paasng/paasng/platform/engine/deploy/bg_command/bkapp_hook_interrupt.py

@@ -0,0 +1,58 @@
+# -*- coding: utf-8 -*-
+# TencentBlueKing is pleased to support the open source community by making
+# 蓝鲸智云 - PaaS 平台 (BlueKing - PaaS System) available.
+# Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
+# Licensed under the MIT License (the "License"); you may not use this file except
+# in compliance with the License. You may obtain a copy of the License at
+#
+#     http://opensource.org/licenses/MIT
+#
+# Unless required by applicable law or agreed to in writing, software distributed under
+# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+# either express or implied. See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# We undertake not to change the open source license (MIT license) applicable
+# to the current version of the project delivered to anyone in the future.
+
+import logging
+
+from paas_wl.bk_app.cnative.specs.constants import (
+    INTERRUPTED_DEPLOY_ID_ANNO_KEY,
+    ApiVersion,
+)
+from paas_wl.core.resource import generate_bkapp_name
+from paas_wl.infras.resources.base import crd
+from paas_wl.infras.resources.base.kres import PatchType
+from paas_wl.infras.resources.utils.basic import get_client_by_app
+from paasng.platform.engine.models.deployment import Deployment
+
+logger = logging.getLogger(__name__)
+
+
+def interrupt_cnative_pre_release(deployment: Deployment) -> None:
+    """中断云原生应用的 PreRelease 阶段.
+
+    :param deployment: 待中断的 Deployment 对象 (云原生应用).
+    """
+    env = deployment.app_environment
+    wl_app = env.wl_app
+    bkapp_name = generate_bkapp_name(env)
+    deploy_id = str(deployment.bkapp_release_id)
+
+    with get_client_by_app(wl_app) as client:
+        # 写入 deploy-interrupted annotation, 通知 operator 协调中断的部署
+        body = {"metadata": {"annotations": {INTERRUPTED_DEPLOY_ID_ANNO_KEY: deploy_id}}}
+        try:
+            crd.BkApp(client, api_version=ApiVersion.V1ALPHA2).patch(
+                name=bkapp_name,
+                namespace=wl_app.namespace,
+                body=body,
+                ptype=PatchType.MERGE,
+            )
+        except Exception:
+            logger.exception(
+                "Failed to patch BkApp deploy-interrupted annotation, bkapp=%s deploy_id=%s",
+                bkapp_name,
+                deploy_id,
+            )

apiserver/paasng/paasng/platform/engine/deploy/interruptions.py

@@ -19,11 +19,14 @@
 from django.utils import timezone
 from django.utils.translation import gettext as _
 
+from paasng.platform.applications.constants import ApplicationType
 from paasng.platform.engine.constants import JobStatus
 from paasng.platform.engine.deploy.bg_build.bg_build import interrupt_build_proc
+from paasng.platform.engine.deploy.bg_command.bkapp_hook_interrupt import interrupt_cnative_pre_release
 from paasng.platform.engine.exceptions import DeployInterruptionFailed
 from paasng.platform.engine.models.deployment import Deployment
 from paasng.platform.engine.workflow import DeploymentCoordinator
+from paasng.platform.modules.constants import DeployHookType
 
 logger = logging.getLogger(__name__)
 
@@ -64,3 +67,21 @@ def interrupt_deployment(deployment: Deployment, user: User):
             # This exception means that build has not been started yet, transform
             # the error message.
             raise DeployInterruptionFailed("任务正处于预备执行状态，无法中断，请稍候重试")
+
+    if _is_cnative_pre_release_phase(deployment):
+        interrupt_cnative_pre_release(deployment)
+
+
+def _is_cnative_pre_release_phase(deployment: Deployment) -> bool:
+    """Return whether the deployment is currently in the cloud-native pre-release phase
+    that is interruptible.
+    """
+    env = deployment.app_environment
+    if env.application.type != ApplicationType.CLOUD_NATIVE:
+        return False
+
+    if deployment.bkapp_release_id is None:
+        return False
+
+    pre_release_hook = env.module.deploy_hooks.get_by_type(DeployHookType.PRE_RELEASE_HOOK)
+    return bool(pre_release_hook and pre_release_hook.enabled)

apiserver/paasng/tests/paasng/platform/engine/deploy/test_bkapp_hook_interrupt.py

@@ -0,0 +1,81 @@
+# -*- coding: utf-8 -*-
+# TencentBlueKing is pleased to support the open source community by making
+# 蓝鲸智云 - PaaS 平台 (BlueKing - PaaS System) available.
+# Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
+# Licensed under the MIT License (the "License"); you may not use this file except
+# in compliance with the License. You may obtain a copy of the License at
+#
+#     http://opensource.org/licenses/MIT
+#
+# Unless required by applicable law or agreed to in writing, software distributed under
+# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+# either express or implied. See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# We undertake not to change the open source license (MIT license) applicable
+# to the current version of the project delivered to anyone in the future.
+
+from contextlib import contextmanager
+from unittest import mock
+
+import pytest
+
+from paas_wl.bk_app.applications.models import WlApp
+from paas_wl.bk_app.cnative.specs.constants import INTERRUPTED_DEPLOY_ID_ANNO_KEY
+from paas_wl.core.resource import generate_bkapp_name
+from paas_wl.infras.resources.base.kres import PatchType
+from paasng.platform.engine.deploy.bg_command.bkapp_hook_interrupt import interrupt_cnative_pre_release
+
+pytestmark = pytest.mark.django_db(databases=["default", "workloads"])
+
+
+class TestInterruptCNativePreRelease:
+    @pytest.fixture(autouse=True)
+    def _setup_wl_app(self, bk_cnative_app, bk_deployment):
+        engine_app = bk_deployment.app_environment.engine_app
+        WlApp.objects.get_or_create(name=engine_app.name, region=engine_app.region)
+        bk_deployment.bkapp_release_id = 1
+        bk_deployment.save(update_fields=["bkapp_release_id", "updated"])
+
+    @pytest.fixture()
+    def mocked_bkapp(self):
+        """patch get_client_by_app 与 crd.BkApp, 返回 BkApp 实例 mock"""
+
+        @contextmanager
+        def fake_get_client(_wl_app):
+            yield mock.MagicMock()
+
+        with (
+            mock.patch(
+                "paasng.platform.engine.deploy.bg_command.bkapp_hook_interrupt.get_client_by_app",
+                side_effect=fake_get_client,
+            ),
+            mock.patch("paasng.platform.engine.deploy.bg_command.bkapp_hook_interrupt.crd.BkApp") as m_bkapp_cls,
+        ):
+            yield m_bkapp_cls.return_value
+
+    def test_interrupt(self, bk_deployment, mocked_bkapp):
+        env = bk_deployment.app_environment
+        wl_app = env.wl_app
+        bkapp_name = generate_bkapp_name(env)
+        deploy_id = str(bk_deployment.bkapp_release_id)
+
+        interrupt_cnative_pre_release(bk_deployment)
+        mocked_bkapp.patch.assert_called_once_with(
+            name=bkapp_name,
+            namespace=wl_app.namespace,
+            body={"metadata": {"annotations": {INTERRUPTED_DEPLOY_ID_ANNO_KEY: deploy_id}}},
+            ptype=PatchType.MERGE,
+        )
+
+    def test_interrupt_best_effort(self, bk_deployment, mocked_bkapp):
+        """patch BkApp 抛异常时, 不应向上抛出, 且会记录日志"""
+        mocked_bkapp.patch.side_effect = RuntimeError("patch failed")
+
+        with mock.patch(
+            "paasng.platform.engine.deploy.bg_command.bkapp_hook_interrupt.logger.exception"
+        ) as mocked_log:
+            interrupt_cnative_pre_release(bk_deployment)
+
+        assert mocked_bkapp.patch.called
+        assert mocked_log.call_count == 1

operator/api/v1alpha2/constants.go

@@ -117,6 +117,10 @@ const (
 	ImageCredentialsRefAnnoKey = "bkapp.paas.bk.tencent.com/image-credentials"
 	// DeployIDAnnoKey 注解中存储 bkpaas 部署ID的键名
 	DeployIDAnnoKey = "bkapp.paas.bk.tencent.com/bkpaas-deploy-id"
+	// InterruptedDeployIDAnnoKey 注解中存储 "最近一次被平台请求中断的部署 ID" 的键名
+	// 取值为该次部署的 deploy_id (字符串), 与 DeployIDAnnoKey 同源. 调和时, 仅当本注解的值
+	// 与 DeployIDAnnoKey 的值同时存在且相等, 才认为"当前部署已被中断"。
+	InterruptedDeployIDAnnoKey = "bkapp.paas.bk.tencent.com/interrupted-deploy-id"
 	// PaaSAnalysisSiteIDAnnoKey 注解中存储 PA site id 的键名
 	PaaSAnalysisSiteIDAnnoKey = "bkapp.paas.bk.tencent.com/paas-analysis-site-id"
 )

operator/go.sum

@@ -225,8 +225,6 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.38.0 h1:jt+WWG8IZlBnVbomuhg2Mdq0+BBQaHbtqHEFEigjUV8=
-golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw=
 golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q=
 golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -246,15 +244,11 @@ golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
-golang.org/x/net v0.40.0 h1:79Xs7wF06Gbdcg4kdCCIQArK11Z1hr5POQ6+fIYHNuY=
-golang.org/x/net v0.40.0/go.mod h1:y0hY0exeL2Pku80/zKK7tpntoX23cqL3Oa6njdgRtds=
 golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY=
 golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.27.0 h1:da9Vo7/tDv5RH/7nZDz1eMGS/q1Vv1N/7FCrBhI9I3M=
 golang.org/x/oauth2 v0.27.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8=
-golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI=
-golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -268,20 +262,14 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw=
-golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
 golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
 golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/term v0.32.0 h1:DR4lr0TjUs3epypdhTOkMmuF5CDFJ/8pOnbzMZPQ7bg=
-golang.org/x/term v0.32.0/go.mod h1:uZG1FhGx848Sqfsq4/DlJr3xGGsYMu/L5GW4abiaEPQ=
 golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU=
 golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4=
-golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA=
 golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM=
 golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM=
 golang.org/x/time v0.10.0 h1:3usCWA8tQn0L8+hFJQNgzpWbd89begxN66o1Ojdn5L4=
@@ -296,8 +284,6 @@ golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roY
 golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
-golang.org/x/tools v0.31.0 h1:0EedkvKDbh+qistFTd0Bcwe/YLh4vHwWEkiI0toFIBU=
-golang.org/x/tools v0.31.0/go.mod h1:naFTU+Cev749tSJRXJlna0T3WxKvb1kWEx15xA4SdmQ=
 golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ=
 golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

operator/pkg/controllers/bkapp/deploy_action.go

@@ -114,11 +114,17 @@ func (r *DeployActionReconciler) Reconcile(ctx context.Context, bkapp *paasv1alp
 
 // validate that there are no running hooks currently, return error if found any running hooks.
 func (r *DeployActionReconciler) validateNoRunningHooks(ctx context.Context, bkapp *paasv1alpha2.BkApp) error {
-	// 如果上一次的部署是被用户主动中断, 则继续执行后续的部署调和流程, 忽略可能处于 progressing 状态的旧 hook
-	// TODO 考虑支持更实时的中断请求, 包括直接删除执行中的 hook 等?
+	// 如果上一次的部署是被用户主动中断, 则继续执行后续的部署调和流程
 	if platdeploy.GetLastDeployStatus(bkapp) == "interrupted" {
 		return nil
 	}
+
+	// 兜底: 平台侧已经写入实时中断信号 (InterruptedDeployIDAnnoKey 指向上次的 DeployId),
+	// 但还没有写入 LastDeployStatusAnnoKey, 通过中断信号的匹配识别上次部署已经被中断
+	if platdeploy.IsDeployInterrupted(bkapp, bkapp.Status.DeployId) {
+		return nil
+	}
+
 	// Check pre-release hook
 	if hookres.IsPreReleaseProgressing(bkapp) {
 		_, err := hooks.CheckAndUpdatePreReleaseHookStatus(

operator/pkg/controllers/bkapp/deploy_action_test.go

@@ -211,5 +211,29 @@ var _ = Describe("Test DeployActionReconciler", func() {
 
 			expectDeployActionInitialized(ret, bkapp, "2")
 		})
+
+		It("deploy ID changed with running hook when previous deploy matches interrupt signal", func() {
+			bkapp.SetAnnotations(
+				map[string]string{
+					paasv1alpha2.DeployIDAnnoKey:            "2",
+					paasv1alpha2.InterruptedDeployIDAnnoKey: "1",
+				},
+			)
+			bkapp.Status.DeployId = "1"
+			bkapp.Status.SetHookStatus(paasv1alpha2.HookStatus{
+				Type:      paasv1alpha2.HookPreRelease,
+				Phase:     paasv1alpha2.HealthProgressing,
+				StartTime: lo.ToPtr(metav1.Now()),
+			})
+
+			hook, err := hookres.BuildPreReleaseHook(bkapp, bkapp.Status.FindHookStatus(paasv1alpha2.HookPreRelease))
+			Expect(err).To(BeNil())
+			Expect(hook.Pod).NotTo(BeNil())
+
+			client := builder.WithObjects(bkapp, hook.Pod).Build()
+			ret := NewDeployActionReconciler(client).Reconcile(context.Background(), bkapp)
+
+			expectDeployActionInitialized(ret, bkapp, "2")
+		})
 	})
 })