feat: add get conflicted config var keys API (#2262)

Author: piglei

apiserver/paasng/paasng/accessories/servicehub/manager.py

@@ -15,11 +15,13 @@
 # We undertake not to change the open source license (MIT license) applicable
 # to the current version of the project delivered to anyone in the future.
 
+import datetime
 import logging
 import operator
 import uuid
 from typing import Callable, Dict, Generator, Iterable, Iterator, List, NamedTuple, Optional, TypeVar, cast
 
+from attrs import define
 from django.http import Http404
 
 from paasng.accessories.servicehub.constants import ServiceBindingType, ServiceType
@@ -237,37 +239,36 @@ def get_env_vars(
         :param filter_enabled: Whether to filter enabled service instances
         :returns: Dict of env variables.
         """
-        rels = list(self.list_provisioned_rels(engine_app, service=service))
-        if filter_enabled:
-            instances = [rel.get_instance() for rel in rels if rel.db_obj.credentials_enabled]
-        else:
-            instances = [rel.get_instance() for rel in rels]
-        # 新的覆盖旧的
-        instances.sort(key=operator.attrgetter("create_time"))
+        groups = self.get_env_var_groups(engine_app, service=service, filter_enabled=filter_enabled)
+        # 如果存在同名环境变量，以较新的为准
+        groups.sort(key=operator.attrgetter("created_at"))
 
         result = {}
-        for i in instances:
-            result.update(i.credentials)
+        for g in groups:
+            result.update(g.data)
         return result
 
-    def get_enabled_env_keys(self, engine_app: EngineApp) -> Dict[str, List[str]]:
-        """
-        Get all provisioned services environment keys
+    def get_env_var_groups(
+        self, engine_app: EngineApp, service: Optional[ServiceObj] = None, filter_enabled: bool = False
+    ) -> "List[EnvVariableGroup]":
+        """Get all provisioned services env variable groups. A group is a collection of environment variables
+        that belong to a specific service instance.
 
         :param engine_app: EngineApp object
-        :return: Dictionary of service display names to list of their environment keys
+        :param service: Optional service object. if given, will only return credentials of the specified service,
+            otherwise return the credentials of all services.
+        :param filter_enabled: Whether to filter enabled service instances
+        :returns: List of env variable groups.
         """
-        provisioned_rels = self.list_provisioned_rels(engine_app)
-        # 凭证的信息写入环境变量的增强服务才展示
-        enabled_rels = [rel for rel in provisioned_rels if rel.db_obj.credentials_enabled]
-
-        results = {}
-        for rel in enabled_rels:
-            # display_name 会被国际化处理为殊字符串类型（__proxy__类型），必须首先将它们转化为标准的字符串
-            service_name = str(rel.get_service().display_name)
-            instance_credentials_keys = list(rel.get_instance().credentials.keys())
-            results[service_name] = instance_credentials_keys
+        results = []
+        for rel in self.list_provisioned_rels(engine_app, service=service):
+            if filter_enabled and not rel.db_obj.credentials_enabled:
+                continue
 
+            inst = rel.get_instance()
+            results.append(
+                EnvVariableGroup(service=rel.get_service(), data=inst.credentials, created_at=inst.create_time)
+            )
         return results
 
     def get_attachment_by_engine_app(self, service: ServiceObj, engine_app: EngineApp):
@@ -340,3 +341,17 @@ def _check_duplicated_sharing(self, service: ServiceObj):
                 raise DuplicatedServiceBoundError(
                     f"Module: {self.module.name} already shared an attachment in service: {service.name}"
                 )
+
+
+@define
+class EnvVariableGroup:
+    """An env variable group.
+
+    :param service: The owner service object.
+    :param data: The environment variables.
+    :param created_at: The time when the var group was created.
+    """
+
+    service: ServiceObj
+    data: Dict[str, str]
+    created_at: datetime.datetime | None

apiserver/paasng/paasng/accessories/servicehub/sharing.py

@@ -24,6 +24,7 @@
 
 from paasng.accessories.servicehub.manager import (
     DuplicatedBindingValidator,
+    EnvVariableGroup,
     SharedServiceInfo,
     get_db_properties,
     get_db_properties_by_service_type,
@@ -130,37 +131,29 @@ def get_env_variables(self, env: ModuleEnvironment, filter_enabled: bool = False
 
         :param env: ModuleEnvironment object, must belongs to self.module
         :param filter_enabled: Whether to filter enabled service instances
+        :return: A dict of environment variables.
         """
-        if env.module != self.module:
-            raise RuntimeError("Invalid env object, must belongs to self.module")
-
         ret = {}
-        for referenced_info in self.list_all_shared_info():
-            ref_env = referenced_info.ref_module.get_envs(env.environment)
-            env_variables = mixed_service_mgr.get_env_vars(ref_env.engine_app, referenced_info.service, filter_enabled)
-            ret.update(env_variables)
+        for g in self.get_env_variable_groups(env, filter_enabled):
+            ret.update(g.data)
         return ret
 
-    def get_enabled_env_keys(self, env: ModuleEnvironment) -> Dict[str, List[str]]:
-        """
-        Retrieve all environment variable keys shared from other modules.
+    def get_env_variable_groups(self, env: ModuleEnvironment, filter_enabled: bool = False) -> List[EnvVariableGroup]:
+        """Get all env variable groups shared from other modules.
 
-        :param env: ModuleEnvironment object that must belong to self.module
-        :return: Dictionary of service display names to their respective env keys list.
+        :param env: ModuleEnvironment object, must belongs to self.module
+        :param filter_enabled: Whether to filter enabled service instances
+        :return: A list of EnvVariableGroup objects.
         """
         if env.module != self.module:
-            raise RuntimeError("Invalid env object, must belong to self.module")
+            raise RuntimeError("Invalid env object, must belongs to self.module")
 
-        result = {}
+        results = []
         for referenced_info in self.list_all_shared_info():
             ref_env = referenced_info.ref_module.get_envs(env.environment)
-            ref_service = referenced_info.service
-            env_keys = mixed_service_mgr.get_env_vars(ref_env.engine_app, ref_service, True)
-            # display_name 会被国际化处理为殊字符串类型（__proxy__类型），必须首先将它们转化为标准的字符串
-            service_name = str(ref_service.display_name)
-            result[service_name] = list(env_keys.keys())
-
-        return result
+            groups = mixed_service_mgr.get_env_var_groups(ref_env.engine_app, referenced_info.service, filter_enabled)
+            results.extend(groups)
+        return results
 
 
 def extract_shared_info(attachment: SharedServiceAttachment) -> Optional[SharedServiceInfo]:

apiserver/paasng/paasng/accessories/servicehub/urls.py

@@ -74,11 +74,6 @@
         views.ModuleServiceAttachmentsViewSet.as_view({"get": "retrieve_info"}),
         name="api.modules.services.info",
     ),
-    re_path(
-        make_app_pattern("/services/config_var_keys/$", include_envs=False),
-        views.ModuleServicesViewSet.as_view({"get": "list_provisioned_env_keys"}),
-        name="api.services.list_provisioned_env_keys",
-    ),
     re_path(
         make_app_pattern(f"/services/{SERVICE_UUID}/possible_plans$", include_envs=False),
         views.ModuleServicesViewSet.as_view({"get": "list_possible_plans"}),

apiserver/paasng/paasng/accessories/servicehub/views.py

@@ -292,20 +292,6 @@ def unbind(self, request, code, module_name, service_id):
         )
         return Response(status=status.HTTP_200_OK)
 
-    def list_provisioned_env_keys(self, request, code, module_name):
-        """获取已经生效的增强服务环境变量 KEY"""
-        module = self.get_module_via_path()
-
-        # env_key_dict 内容示例： {"svc_name": ["key1", "key2"]}
-        env_key_dict: Dict[str, List[str]] = {}
-        for env in module.get_envs():
-            env_key_dict = {
-                **env_key_dict,
-                **ServiceSharingManager(env.module).get_enabled_env_keys(env),
-                **mixed_service_mgr.get_enabled_env_keys(env.engine_app),
-            }
-        return Response(data=env_key_dict)
-
 
 class ServiceViewSet(viewsets.ViewSet, ApplicationCodeInPathMixin):
     """增强服务相关视图(与应用无关的)"""

apiserver/paasng/paasng/platform/bkapp_model/manifest.py

@@ -74,7 +74,7 @@
     merge_env_vars_overlay,
     override_env_vars_overlay,
 )
-from paasng.platform.engine.configurations.config_var import get_env_variables
+from paasng.platform.engine.configurations.config_var import EnvVarSource, UnifiedEnvVarsReader
 from paasng.platform.engine.constants import AppEnvName, ConfigVarEnvName, RuntimeType
 from paasng.platform.engine.models import Deployment
 from paasng.platform.engine.models.config_var import ENVIRONMENT_ID_FOR_GLOBAL, ConfigVar
@@ -592,9 +592,18 @@ def apply_builtin_env_vars(model_res: crd.BkAppResource, env: ModuleEnvironment)
 
     # 此处，云原生应用忽略这些类型的环境变量：用户手动定义、描述文件定义、服务发现，
     # 忽略用户手动定义（include_config_vars）是因为 EnvVarsManifestConstructor 已处理过。
-    for name, value in get_env_variables(
-        env, include_config_vars=False, include_preset_env_vars=False, include_svc_disc=False
-    ).items():
+    system_vars = UnifiedEnvVarsReader(env).get_kv_map(
+        exclude_sources=[
+            # 用户在产品或描述文件手动定义，已经由 EnvVarsManifestConstructor 处理
+            EnvVarSource.USER_CONFIGURED,
+            EnvVarSource.USER_PRESET,
+            # 服务发现环境变量，已经由 SvcDiscoveryManifestConstructor 处理，无需在模型主动添加环境变量
+            EnvVarSource.BUILTIN_SVC_DISC,
+            # 云原生应用不需要在构建镜像中推送制品到 blobstore，因此忽略
+            EnvVarSource.BUILTIN_BLOBSTORE,
+        ]
+    )
+    for name, value in system_vars.items():
         env_vars.append(crd.EnvVar(name=name, value=value))
         builtin_env_vars_overlay.append(crd.EnvVarOverlay(envName=environment, name=name, value=value))