feat: add validate for source repo url (#2318)

Author: jamesgetx

apiserver/paasng/conf.yaml.tpl

@@ -939,6 +939,10 @@ BK_AUDIT_ENDPOINT = ""
 #         - example-auth-token
 #       host: http://bkmonitor-query.example.com
 
+## ------------------------------------ 安全配置 ------------------------------------
+
+# FORBIDDEN_REPO_PORTS 包含与代码/镜像仓库相关的敏感端口，配置后，平台将不允许用户填写或注册相关的代码/镜像仓库
+FORBIDDEN_REPO_PORTS = [21, 22, 23]
 
 ## ------------------------------------ internal 配置，仅开发项目与特殊环境下使用 ------------------------------------
 

apiserver/paasng/paasng/platform/modules/serializers.py

@@ -44,7 +44,14 @@
 from paasng.platform.templates.models import Template
 from paasng.utils.i18n.serializers import TranslatedCharField
 from paasng.utils.serializers import SourceControlField, UserNameField
-from paasng.utils.validators import RE_APP_CODE, DnsSafeNameValidator, ReservedWordValidator, validate_procfile
+from paasng.utils.validators import (
+    RE_APP_CODE,
+    DnsSafeNameValidator,
+    ReservedWordValidator,
+    validate_image_repo,
+    validate_procfile,
+    validate_repo_url,
+)
 
 
 def validate_build_method(build_method: RuntimeType, source_origin: SourceOrigin):
@@ -272,6 +279,22 @@ def validate_source_init_template(self, tmpl_name):
             raise ValidationError(_("模板 {} 不可用").format(tmpl_name))
         return tmpl_name
 
+    def validate(self, attrs):
+        source_repo_url = attrs.get("source_repo_url")
+        if source_repo_url:
+            self._validate_source_repo_url(source_repo_url, attrs["source_origin"])
+        return attrs
+
+    @staticmethod
+    def _validate_source_repo_url(source_repo_url, source_origin):
+        try:
+            if source_origin == SourceOrigin.CNATIVE_IMAGE:
+                validate_image_repo(source_repo_url)
+            else:
+                validate_repo_url(source_repo_url)
+        except ValueError as e:
+            raise ValidationError({"source_repo_url": str(e)})
+
 
 class ModuleBuildConfigSLZ(serializers.Serializer):
     """模块镜像构建信息"""

apiserver/paasng/paasng/platform/sourcectl/serializers.py

@@ -18,6 +18,7 @@
 import logging
 
 from rest_framework import serializers
+from rest_framework.exceptions import ValidationError
 from rest_framework.validators import UniqueTogetherValidator
 
 from paasng.platform.applications.serializers.fields import SourceDirField
@@ -26,6 +27,7 @@
 from paasng.platform.sourcectl.source_types import get_sourcectl_type
 from paasng.platform.sourcectl.type_specs import BkSvnSourceTypeSpec
 from paasng.utils.serializers import SourceControlField, UserNameField, VerificationCodeField
+from paasng.utils.validators import validate_image_repo, validate_repo_url
 
 logger = logging.getLogger(__name__)
 
@@ -170,6 +172,22 @@ class RepoBackendModifySLZ(serializers.Serializer):
     source_repo_auth_info = serializers.JSONField(required=False, default={})
     source_dir = SourceDirField(help_text="Procfile 所在目录, 如果是根目录可不填.")
 
+    def validate_source_repo_url(self, value: str) -> str:
+        if not value:
+            return value
+
+        is_image_repo = self.context["is_image_repo"]
+
+        try:
+            if is_image_repo:
+                validate_image_repo(value)
+            else:
+                validate_repo_url(value)
+        except ValueError as e:
+            raise ValidationError(str(e))
+        else:
+            return value
+
 
 ##########################
 # 源码包相关的 serializers #

apiserver/paasng/paasng/platform/sourcectl/views.py

@@ -358,7 +358,7 @@ def modify(self, request, code, module_name):
         if module.get_source_origin() == SourceOrigin.IMAGE_REGISTRY:
             return self._modify_image(request, code, module_name)
 
-        slz = slzs.RepoBackendModifySLZ(data=self.request.data)
+        slz = slzs.RepoBackendModifySLZ(data=self.request.data, context={"is_image_repo": False})
         slz.is_valid(raise_exception=True)
         data = slz.data
         repo_type = data["source_control_type"]
@@ -408,7 +408,7 @@ def modify(self, request, code, module_name):
 
     def _modify_image(self, request, code, module_name):
         module = self.get_module_via_path()
-        slz = slzs.RepoBackendModifySLZ(data=request.data)
+        slz = slzs.RepoBackendModifySLZ(data=request.data, context={"is_image_repo": True})
         slz.is_valid(raise_exception=True)
         data = slz.data
         repo_url = data["source_repo_url"]

apiserver/paasng/paasng/settings/__init__.py

@@ -1538,3 +1538,6 @@ def _build_file_handler(log_path: Path, filename: str, format: str) -> Dict:
 FE_FEATURE_SETTINGS_CNATIVE_MGRLEGACY = settings.get("FE_FEATURE_SETTINGS_CNATIVE_MGRLEGACY", False)
 # 应用令牌，用于 APP 调用用户态的云 API
 FE_FEATURE_SETTINGS_APP_ACCESS_TOKEN = settings.get("FE_FEATURE_SETTINGS_APP_ACCESS_TOKEN", False)
+
+# FORBIDDEN_REPO_PORTS 包含与代码/镜像仓库相关的敏感端口，配置后，平台将不允许用户填写或注册相关的代码/镜像仓库
+FORBIDDEN_REPO_PORTS = settings.get("FORBIDDEN_REPO_PORTS", [])

apiserver/paasng/paasng/utils/validators.py

@@ -18,12 +18,15 @@
 import base64
 import re
 from typing import Dict
+from urllib.parse import urlparse
 
+from django.conf import settings
 from django.core.exceptions import ValidationError
 from django.core.validators import RegexValidator
 from django.utils.deconstruct import deconstructible
 from django.utils.encoding import force_str
 from past.builtins import basestring
+from moby_distribution.registry.utils import parse_image
 
 from paasng.core.region.models import Region, RegionList, filter_region_by_name
 
@@ -158,3 +161,48 @@ def validate_procfile(procfile: Dict[str, str]) -> Dict[str, str]:
 
     # Formalize procfile data and return
     return {k.lower(): v for k, v in procfile.items()}
+
+
+def validate_image_repo(image_repo: str):
+    """Validate image repo port security.
+
+    :param image_repo: image repo
+    :raise: ValueError if image repo is invalid
+    """
+    repo_domain = parse_image(image_repo, default_registry="docker.io").domain
+
+    if ":" not in repo_domain:
+        return
+
+    repo_port = repo_domain.rsplit(":")[-1]
+
+    try:
+        port = int(repo_port)
+    except ValueError:
+        raise ValueError(f"Invalid image repo: the port {repo_port} is not an integer")
+
+    if port in [int(p) for p in settings.FORBIDDEN_REPO_PORTS]:
+        raise ValueError(f"Invalid image repo: the port number {port} is forbidden")
+
+
+def validate_repo_url(repo_url: str):
+    """Validate repo url format, protocol, and port security.
+
+    :param repo_url: repo url
+    :raise: ValueError if repo url is invalid
+    """
+    try:
+        parsed_url = urlparse(repo_url)
+    except Exception:
+        raise ValueError("Invalid url")
+
+    if not parsed_url.netloc:
+        parsed_url = urlparse(f"https://{repo_url}")
+        if not parsed_url.netloc:
+            raise ValueError("Invalid url")
+
+    if parsed_url.scheme not in ["http", "https", "git", "svn"]:
+        raise ValueError("Invalid url: only support http/https/git/svn scheme")
+
+    if parsed_url.port and parsed_url.port in [int(p) for p in settings.FORBIDDEN_REPO_PORTS]:
+        raise ValueError(f"Invalid url: the port number {parsed_url.port} is forbidden")

apiserver/paasng/tests/paasng/test_utils/test_validators.py

@@ -18,7 +18,7 @@
 import pytest
 from django.core.exceptions import ValidationError
 
-from paasng.utils.validators import DnsSafeNameValidator, ReservedWordValidator
+from paasng.utils.validators import DnsSafeNameValidator, ReservedWordValidator, validate_image_repo, validate_repo_url
 
 
 class TestReservedWordValidator:
@@ -47,3 +47,64 @@ def test_negative_sample(self, input_str):
         with pytest.raises(ValidationError) as exec_info:
             validator(input_str)
         assert exec_info.value.message == validator.message
+
+
+class Test__validate_repo_url:
+    @pytest.mark.parametrize(
+        "repo_url",
+        ["mysql://127.0.0.1", "postgres://127.0.0.1"],
+    )
+    def test_invalid_protocol(self, repo_url):
+        with pytest.raises(ValueError, match="Invalid url: only support http/https/git/svn scheme"):
+            validate_repo_url(repo_url)
+
+    @pytest.mark.parametrize(
+        "repo_url",
+        ["http://127.0.0.1:22/bkapps.git", "https://127.0.0.1:23/bkapps.git"],
+    )
+    def test_invalid_port(self, repo_url, settings):
+        settings.FORBIDDEN_REPO_PORTS = [22, 23]
+        with pytest.raises(ValueError, match=r"Invalid url: the port number \d+ is forbidden"):
+            validate_repo_url(repo_url)
+
+    @pytest.mark.parametrize(
+        "repo_url",
+        ["/www.example.com", "//127.0.0.1"],
+    )
+    def test_invalid_url(self, repo_url):
+        with pytest.raises(ValueError, match="Invalid url"):
+            validate_repo_url(repo_url)
+
+    @pytest.mark.parametrize(
+        "repo_url",
+        [
+            "http://127.0.0.1:80/bkapps.git",
+            "https://127.0.0.1/bkapps.git",
+            "git://127.0.0.1",
+            "svn://127.0.0.1",
+        ],
+    )
+    def test_valid_url(self, repo_url):
+        validate_repo_url(repo_url)
+
+
+class Test__validate_image_repo:
+    @pytest.mark.parametrize(
+        "image_repo",
+        ["mirror.tencent.com:22/bkpaas", "mirror.tencent.com:23/bkapps"],
+    )
+    def test_invalid_port(self, image_repo, settings):
+        settings.FORBIDDEN_REPO_PORTS = [22, 23]
+        with pytest.raises(ValueError, match=r"Invalid image repo: the port number \d+ is forbidden"):
+            validate_image_repo(image_repo)
+
+    @pytest.mark.parametrize(
+        "image_repo",
+        [
+            "mirror.tencent.com/bkapps",
+            "mirror.tencent.com:443/bkapps",
+            "nginx",
+        ],
+    )
+    def test_valid_repo(self, image_repo):
+        validate_image_repo(image_repo)