feat: bklog 支持多租户配置 (#2772)

Author: xiaoj655

apiserver/paasng/conf.yaml.tpl

@@ -696,17 +696,6 @@ BK_CI_CLIENT_USERNAME = "blueking"
 # ENABLE_BK_LOG_APIGW: True
 # 蓝鲸日志平台网关的环境，仅在 ENABLE_BK_LOG_APIGW=True 时生效
 # BK_LOG_APIGW_SERVICE_STAGE: stag
-# 日志保存时间（天数），默认值 14
-# BKLOG_RETENTION: 14
-# Elasticsearch 索引分片数，默认值 1
-# BKLOG_ES_SHARDS: 1
-# 存储副本数，默认值 1
-# BKLOG_STORAGE_REPLICAS: 1
-# 蓝鲸日志平台相关的配置项 （当设置此项时，上述单独的 BKLOG_* 配置项将被覆盖）
-# BKLOG_CONFIG:
-#   RETENTION: 14
-#   ES_SHARDS: 1
-#   STORAGE_REPLICAS: 1
 
 
 ## ------------------------------------ 蓝鲸通知中心配置 ------------------------------------

apiserver/paasng/locale/en/LC_MESSAGES/django.po

@@ -705,6 +705,26 @@ msgstr "Is Built-in Collection Item"
 msgid "是否启用"
 msgstr "Is Enabled"
 
+#: paasng/accessories/log/models.py:208
+msgid "日志平台存储集群 ID"
+msgstr "Blueking log platform Storage Cluster ID"
+
+#: paasng/accessories/log/models.py:209
+msgid "日志保存时间（天）"
+msgstr "Log Retention Period (Days)"
+
+#: paasng/accessories/log/models.py:210
+msgid "ES 索引分片数"
+msgstr "Number of ES Index Shards"
+
+#: paasng/accessories/log/models.py:211
+msgid "存储副本数"
+msgstr "Number of Storage Replicas"
+
+#: paasng/accessories/log/models.py:214
+msgid "时区"
+msgstr "Time Zone"
+
 #: paasng/accessories/log/serializers.py:156
 #, python-brace-format
 msgid "最多仅能查看前 {MAX_RESULT_WINDOW} 条日志"

apiserver/paasng/paasng/accessories/log/exceptions.py

@@ -54,3 +54,11 @@ class BkLogApiError(BkLogGatewayServiceError):
     """When calling the bk_log api, bk_log returns an error message,
     which needs to be captured and displayed to the user on the page
     """
+
+
+class TenantLogConfigNotFoundError(Exception):
+    """该租户日志配置不存在"""
+
+    def __init__(self, tenant_id: str):
+        self.message = f"TenantLogConfig not found for tenant_id: {tenant_id}"
+        super().__init__(self.message)

apiserver/paasng/paasng/accessories/log/management/commands/create_tenant_log_config.py

@@ -0,0 +1,86 @@
+# -*- coding: utf-8 -*-
+# TencentBlueKing is pleased to support the open source community by making
+# 蓝鲸智云 - PaaS 平台 (BlueKing - PaaS System) available.
+# Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
+# Licensed under the MIT License (the "License"); you may not use this file except
+# in compliance with the License. You may obtain a copy of the License at
+#
+#     http://opensource.org/licenses/MIT
+#
+# Unless required by applicable law or agreed to in writing, software distributed under
+# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+# either express or implied. See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# We undertake not to change the open source license (MIT license) applicable
+# to the current version of the project delivered to anyone in the future.
+
+from django.core.exceptions import ValidationError
+from django.core.management.base import BaseCommand, CommandError
+
+from paasng.accessories.log.models import TenantLogConfig
+from paasng.core.tenant.user import get_init_tenant_id
+
+
+class Command(BaseCommand):
+    help = "Create TenantLogConfig for a tenant"
+    config_fields = (
+        "storage_cluster_id",
+        "retention",
+        "es_shards",
+        "storage_replicas",
+        "time_zone",
+    )
+
+    def add_arguments(self, parser):
+        tenant_group = parser.add_mutually_exclusive_group(required=True)
+        tenant_group.add_argument("--tenant-id", help="tenant id")
+        tenant_group.add_argument(
+            "--default-tenant",
+            action="store_true",
+            default=False,
+            help="为默认租户创建配置，不可与 --tenant-id 同时指定",
+        )
+        parser.add_argument(
+            "--update",
+            action="store_true",
+            default=False,
+            help="允许更新相同 tenant_id 已存在的配置",
+        )
+        parser.add_argument("--storage-cluster-id", required=True, type=int, help="日志平台存储集群 ID")
+        parser.add_argument("--retention", type=int, default=14, help="日志保存时间（天），默认 14")
+        parser.add_argument("--es-shards", type=int, default=1, help="ES 索引分片数，默认 1")
+        parser.add_argument("--storage-replicas", type=int, default=1, help="存储副本数，默认 1")
+        parser.add_argument("--time-zone", type=int, default=8, help="时区，如 8 表示 UTC+8，默认 8")
+
+    def handle(self, *args, **options):
+        tenant_id = get_init_tenant_id() if options["default_tenant"] else options["tenant_id"]
+        config_data = {field: options[field] for field in self.config_fields}
+        allow_update = options["update"]
+        existing = TenantLogConfig.objects.filter(tenant_id=tenant_id).first()
+
+        if existing and not allow_update:
+            raise CommandError(f"TenantLogConfig already exists for tenant_id={tenant_id}, use --update to overwrite")
+
+        try:
+            config = existing or TenantLogConfig(tenant_id=tenant_id)
+            for field, value in config_data.items():
+                setattr(config, field, value)
+            action = "Updated" if existing else "Created"
+
+            config.full_clean()
+            if existing:
+                config.save(update_fields=[*config_data.keys(), "updated"])
+            else:
+                config.save()
+        except ValidationError as e:
+            raise CommandError(f"Invalid TenantLogConfig data: {e}")
+
+        self.stdout.write(
+            self.style.SUCCESS(
+                f"{action} TenantLogConfig for tenant_id={tenant_id}: "
+                f"storage_cluster_id={config.storage_cluster_id}, retention={config.retention}, "
+                f"es_shards={config.es_shards}, storage_replicas={config.storage_replicas}, "
+                f"time_zone={config.time_zone}"
+            )
+        )

apiserver/paasng/paasng/accessories/log/migrations/0008_tenantlogconfig.py

@@ -0,0 +1,31 @@
+# Generated by Django 4.2.23 on 2026-04-07 08:28
+
+from django.db import migrations, models
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('log', '0007_customcollectorconfig_tenant_id_and_more'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='TenantLogConfig',
+            fields=[
+                ('uuid', models.UUIDField(auto_created=True, default=uuid.uuid4, editable=False, primary_key=True, serialize=False, unique=True, verbose_name='UUID')),
+                ('created', models.DateTimeField(auto_now_add=True)),
+                ('updated', models.DateTimeField(auto_now=True)),
+                ('tenant_id', models.CharField(default='default', help_text='本条数据的所属租户', max_length=32, unique=True, verbose_name='租户 ID')),
+                ('storage_cluster_id', models.IntegerField(help_text='指定该租户日志存储的 ES 集群', verbose_name='日志平台存储集群 ID')),
+                ('retention', models.IntegerField(verbose_name='日志保存时间（天）')),
+                ('es_shards', models.IntegerField(verbose_name='ES 索引分片数')),
+                ('storage_replicas', models.IntegerField(verbose_name='存储副本数')),
+                ('time_zone', models.IntegerField(help_text='日志时间字段解析时区，如 8 代表 UTC+8', verbose_name='时区')),
+            ],
+            options={
+                'abstract': False,
+            },
+        ),
+    ]

apiserver/paasng/paasng/accessories/log/migrations/0009_init_tenantlogconfig_from_deprecated_bklog_config.py

@@ -0,0 +1,84 @@
+# -*- coding: utf-8 -*-
+# TencentBlueKing is pleased to support the open source community by making
+# 蓝鲸智云 - PaaS 平台 (BlueKing - PaaS System) available.
+# Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
+# Licensed under the MIT License (the "License"); you may not use this file except
+# in compliance with the License. You may obtain a copy of the License at
+#
+#     http://opensource.org/licenses/MIT
+#
+# Unless required by applicable law or agreed to in writing, software distributed under
+# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+# either express or implied. See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# We undertake not to change the open source license (MIT license) applicable
+# to the current version of the project delivered to anyone in the future.
+
+# 历史兼容 migration, 从已被废弃的 BKLOG_CONFIG 配置项, 初始化为默认租户的 TenantLogConfig 记录
+
+import datetime
+
+from paasng.settings import settings
+from django.db import migrations
+from django.utils.timezone import get_default_timezone
+from paasng.core.tenant.user import get_init_tenant_id
+
+BKLOG_TIME_ZONE = settings.get("BKLOG_TIME_ZONE")
+## 日志平台存储集群 ID
+BKLOG_STORAGE_CLUSTER_ID = settings.get("BKLOG_STORAGE_CLUSTER_ID")
+## 日志保存时间（天数），默认值 14
+BKLOG_RETENTION = int(settings.get("BKLOG_RETENTION", 14))
+## Elasticsearch 索引分片数，默认值 1
+BKLOG_ES_SHARDS = int(settings.get("BKLOG_ES_SHARDS", 1))
+## 存储副本数，默认值 1
+BKLOG_STORAGE_REPLICAS = int(settings.get("BKLOG_STORAGE_REPLICAS", 1))
+BKLOG_CONFIG = settings.get(
+    "BKLOG_CONFIG",
+    {
+        "TIME_ZONE": BKLOG_TIME_ZONE,
+        "STORAGE_CLUSTER_ID": BKLOG_STORAGE_CLUSTER_ID,
+        "RETENTION": BKLOG_RETENTION,
+        "ES_SHARDS": BKLOG_ES_SHARDS,
+        "STORAGE_REPLICAS": BKLOG_STORAGE_REPLICAS,
+    },
+)
+
+
+def get_time_zone() -> int:
+    # 历史兼容，默认使用 django 的时区设置
+    time_zone = BKLOG_CONFIG.get("TIME_ZONE")
+    if time_zone is not None:
+        return time_zone
+
+    tz = get_default_timezone()
+    return int(tz.utcoffset(datetime.datetime.now()).total_seconds() // 60 // 60)
+
+
+def init_tenant_log_config(apps, schema_editor):
+    tenant_log_config = apps.get_model("log", "TenantLogConfig")
+    storage_cluster_id = BKLOG_CONFIG.get("STORAGE_CLUSTER_ID")
+
+    if storage_cluster_id is None:
+        return
+
+    tenant_log_config.objects.get_or_create(
+        tenant_id=get_init_tenant_id(),
+        defaults={
+            "storage_cluster_id": storage_cluster_id,
+            "retention": BKLOG_CONFIG.get("RETENTION"),
+            "es_shards": BKLOG_CONFIG.get("ES_SHARDS"),
+            "storage_replicas": BKLOG_CONFIG.get("STORAGE_REPLICAS"),
+            "time_zone": get_time_zone(),
+        },
+    )
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("log", "0008_tenantlogconfig"),
+    ]
+
+    operations = [
+        migrations.RunPython(init_tenant_log_config, reverse_code=migrations.RunPython.noop),
+    ]

apiserver/paasng/paasng/accessories/log/models.py

@@ -194,3 +194,24 @@ class CustomCollectorConfig(UuidAuditedModel):
 
     class Meta:
         unique_together = ("module", "name_en")
+
+
+class TenantLogConfig(UuidAuditedModel):
+    """租户日志配置
+
+    每个租户必须有一条配置记录才能使用日志平台方案
+    可以使用 `python manage.py create_tenant_log_config` 命令创建此配置
+    """
+
+    tenant_id = tenant_id_field_factory(unique=True)
+
+    # ------------------- 以下字段严格匹配 bk_log 平台 ------------------ #
+    storage_cluster_id = models.IntegerField(_("日志平台存储集群 ID"), help_text="指定该租户日志存储的 ES 集群")
+    retention = models.IntegerField(_("日志保存时间（天）"))
+    es_shards = models.IntegerField(_("ES 索引分片数"))
+    storage_replicas = models.IntegerField(_("存储副本数"))
+
+    time_zone = models.IntegerField(
+        _("时区"),
+        help_text="日志时间字段解析时区，如 8 代表 UTC+8",
+    )

apiserver/paasng/paasng/accessories/log/shim/setup_bklog.py

@@ -15,21 +15,22 @@
 # We undertake not to change the open source license (MIT license) applicable
 # to the current version of the project delivered to anyone in the future.
 
-import datetime
 import logging
+from functools import cached_property
 from typing import Union
 
 from django.conf import settings
-from django.utils.timezone import get_default_timezone
 from django.utils.translation import gettext_lazy as _
 
 from paas_wl.infras.cluster.shim import EnvClusterService
 from paasng.accessories.log.constants import DEFAULT_LOG_CONFIG_PLACEHOLDER
+from paasng.accessories.log.exceptions import TenantLogConfigNotFoundError
 from paasng.accessories.log.models import CustomCollectorConfig as CustomCollectorConfigModel
 from paasng.accessories.log.models import (
     ElasticSearchConfig,
     ElasticSearchParams,
     ProcessLogQueryConfig,
+    TenantLogConfig,
 )
 from paasng.accessories.log.shim.bklog_custom_collector_config import get_or_create_custom_collector_config
 from paasng.accessories.log.shim.setup_elk import ELK_INGRESS_COLLECTOR_CONFIG_ID_TMPL, setup_platform_elk_config
@@ -54,32 +55,41 @@
 class BKLogConfigProvider:
     def __init__(self, module: Module):
         self.module = module
+        self.tenant_id = module.application.tenant_id
+
+    @cached_property
+    def config(self):
+        """获取租户的日志配置"""
+
+        try:
+            return TenantLogConfig.objects.get(tenant_id=self.tenant_id)
+        except TenantLogConfig.DoesNotExist:
+            raise TenantLogConfigNotFoundError(self.tenant_id)
 
     @property
     def timezone(self) -> int:
-        if timezone := settings.BKLOG_CONFIG.get("TIME_ZONE"):
-            return timezone
-        tz = get_default_timezone()
-        return int(tz.utcoffset(datetime.datetime.now()).total_seconds() // 60 // 60)
+        """获取时区"""
+        return self.config.time_zone
 
     @property
     def storage_cluster_id(self) -> int:
-        return settings.BKLOG_CONFIG["STORAGE_CLUSTER_ID"]
+        """获取存储集群 ID（从 TenantLogConfig）"""
+        return self.config.storage_cluster_id
 
     @property
     def retention(self) -> int:
-        """获取日志存储时间(天)"""
-        return settings.BKLOG_CONFIG["RETENTION"]
+        """获取日志保存时间（从 TenantLogConfig）"""
+        return self.config.retention
 
     @property
     def es_shards(self) -> int:
-        """获取ES索引分片数"""
-        return settings.BKLOG_CONFIG["ES_SHARDS"]
+        """获取 ES 索引分片数（从 TenantLogConfig）"""
+        return self.config.es_shards
 
     @property
     def storage_replicas(self) -> int:
-        """获取存储副本数"""
-        return settings.BKLOG_CONFIG["STORAGE_REPLICAS"]
+        """获取存储副本数（从 TenantLogConfig）"""
+        return self.config.storage_replicas
 
 
 def _add_wildcard_suffix(path: str) -> str:

apiserver/paasng/paasng/settings/__init__.py

@@ -1179,26 +1179,6 @@ def _build_file_handler(log_path: Path, filename: str, format: str) -> Dict:
 ENABLE_BK_LOG_APIGW = settings.get("ENABLE_BK_LOG_APIGW", True)
 # 蓝鲸日志平台网关的环境，仅在 ENABLE_BK_LOG_APIGW=True 时生效
 BK_LOG_APIGW_SERVICE_STAGE = settings.get("BK_LOG_APIGW_SERVICE_STAGE", "stag")
-# 蓝鲸日志平台相关的配置项
-BKLOG_TIME_ZONE = settings.get("BKLOG_TIME_ZONE")
-## 日志平台存储集群 ID
-BKLOG_STORAGE_CLUSTER_ID = settings.get("BKLOG_STORAGE_CLUSTER_ID")
-## 日志保存时间（天数），默认值 14
-BKLOG_RETENTION = int(settings.get("BKLOG_RETENTION", 14))
-## Elasticsearch 索引分片数，默认值 1
-BKLOG_ES_SHARDS = int(settings.get("BKLOG_ES_SHARDS", 1))
-## 存储副本数，默认值 1
-BKLOG_STORAGE_REPLICAS = int(settings.get("BKLOG_STORAGE_REPLICAS", 1))
-BKLOG_CONFIG = settings.get(
-    "BKLOG_CONFIG",
-    {
-        "TIME_ZONE": BKLOG_TIME_ZONE,
-        "STORAGE_CLUSTER_ID": BKLOG_STORAGE_CLUSTER_ID,
-        "RETENTION": BKLOG_RETENTION,
-        "ES_SHARDS": BKLOG_ES_SHARDS,
-        "STORAGE_REPLICAS": BKLOG_STORAGE_REPLICAS,
-    },
-)
 
 # 日志 ES 服务地址
 ELASTICSEARCH_HOSTS = settings.get(