feat: enhance file upload, git url validation and command handling (#2835)

Co-authored-by: piglei <piglei2007@gmail.com>

Author: narasux

apiserver/paasng/paasng/misc/tools/smart_app/serializers.py

@@ -24,14 +24,15 @@
 from paasng.utils.i18n.serializers import TranslatedCharField
 from paasng.utils.models import OrderByField
 from paasng.utils.serializers import StringArrayField
+from paasng.utils.validators import SafeFilenameValidator
 
 from .constants import SourceCodeOriginType
 
 
 class ToolPackageStashInputSLZ(serializers.Serializer):
     """Upload source package SLZ"""
 
-    package = serializers.FileField(help_text="待构建的应用源码包")
+    package = serializers.FileField(help_text="待构建的应用源码包", validators=[SafeFilenameValidator()])
 
 
 class BaseSmartBuildSLZ(serializers.Serializer):

apiserver/paasng/paasng/misc/tools/smart_app/views.py

@@ -36,7 +36,7 @@
 from paasng.platform.smart_app.services.prepared import PreparedSourcePackage
 from paasng.platform.sourcectl.utils import generate_temp_dir
 from paasng.utils.error_codes import error_codes
-from paasng.utils.views import get_filepath
+from paasng.utils.views import save_uploaded_file
 
 from .build import SmartBuildCoordinator, SmartBuildTaskRunner, create_smart_build_record
 from .filters import SmartBuildRecordFilterBackend
@@ -72,7 +72,7 @@ def upload(self, request):
         slz.is_valid(raise_exception=True)
 
         with generate_temp_dir() as tmp_dir:
-            filepath = get_filepath(slz.validated_data["package"], str(tmp_dir))
+            filepath = save_uploaded_file(slz.validated_data["package"], str(tmp_dir))
             stat = SourcePackageStatReader(filepath).read()
 
             if not stat.meta_info:

apiserver/paasng/paasng/platform/smart_app/serializers.py

@@ -15,9 +15,6 @@
 # We undertake not to change the open source license (MIT license) applicable
 # to the current version of the project delivered to anyone in the future.
 
-import re
-
-from django.utils.translation import gettext_lazy as _
 from rest_framework import serializers
 
 from paasng.core.tenant.constants import AppTenantMode
@@ -26,6 +23,7 @@
 from paasng.platform.declarative.application.validations.v2 import MarketSLZ, ModuleDescriptionSLZ
 from paasng.platform.declarative.constants import DiffType
 from paasng.utils.i18n.serializers import TranslatedCharField
+from paasng.utils.validators import SafeFilenameValidator
 
 
 class AppDescriptionSLZ(serializers.Serializer):
@@ -41,18 +39,19 @@ class AppDescriptionSLZ(serializers.Serializer):
 class PackageStashRequestSLZ(serializers.Serializer):
     """Handle S-mart application uploads"""
 
-    package = serializers.FileField(help_text="应用源码包")
+    package = serializers.FileField(
+        help_text="应用源码包",
+        validators=[
+            SafeFilenameValidator(
+                name_pattern=r"[a-zA-Z0-9\-_.]+",
+                name_pattern_message="格式错误，只能包含字母(a-zA-Z)、数字(0-9)和半角连接符(-)、下划线(_)和点(.)",
+            )
+        ],
+    )
     app_tenant_mode = serializers.ChoiceField(
         help_text="应用租户模式", choices=AppTenantMode.get_choices(), default=None
     )
 
-    def validate_package(self, package):
-        if not re.fullmatch("[a-zA-Z0-9-_. ]+", package.name):
-            raise serializers.ValidationError(
-                {"invalid": _("格式错误，只能包含字母(a-zA-Z)、数字(0-9)和半角连接符(-)、下划线(_)、空格( )和点(.)")}
-            )
-        return package
-
 
 class PackageStashConfirmRequestSLZ(AppBasicInfoMixin):
     """Handle S-mart application confirm after upload

apiserver/paasng/paasng/platform/smart_app/views.py

@@ -68,7 +68,7 @@
 from paasng.platform.sourcectl.utils import generate_temp_dir, generate_temp_file
 from paasng.utils.error_codes import error_codes
 from paasng.utils.moby_distribution.registry.exceptions import RequestError as RequestRegistryError
-from paasng.utils.views import get_filepath
+from paasng.utils.views import save_uploaded_file
 
 logger = logging.getLogger(__name__)
 
@@ -109,7 +109,7 @@ def upload(self, request):
         )
 
         with generate_temp_dir() as download_dir:
-            filepath = get_filepath(package_fp, str(download_dir))
+            filepath = save_uploaded_file(package_fp, str(download_dir))
 
             stat = SourcePackageStatReader(filepath).read()
 
@@ -321,7 +321,7 @@ def stash(self, request, code):
         )
 
         with generate_temp_dir() as download_dir:
-            filepath = get_filepath(package_fp, download_dir)
+            filepath = save_uploaded_file(package_fp, download_dir)
 
             stat = SourcePackageStatReader(filepath).read()
 

apiserver/paasng/paasng/platform/sourcectl/git/client.py

@@ -29,6 +29,7 @@
 from django.utils.encoding import force_str
 
 from paasng.platform.sourcectl.exceptions import RequestTimeOutError
+from paasng.utils.validators import validate_repo_url
 
 logger = logging.getLogger(__name__)
 
@@ -37,6 +38,10 @@ class GitCommandExecutionError(Exception):
     """执行 git 命令时发生错误"""
 
 
+class GitURLValidationError(GitCommandExecutionError):
+    """Git 仓库 URL 安全校验失败"""
+
+
 @dataclass
 class Ref:
     name: str
@@ -50,22 +55,36 @@ def __str__(self):
 
 
 class GitCommand:
+    """Git 命令封装。
+
+    :param args: Git 子命令选项参数，位于 ``--`` 之前。
+    :param end_of_options_args: ``--`` 之后的参数（URL、路径、分支名等），
+        这些参数不会被 git 解释为选项，即使它们以 ``-`` 开头。
+        构建命令时会自动在前面插入 ``--`` 分隔符。
+    """
+
     def __init__(
         self,
         git_filepath: str,
         command: str,
-        args: Optional[List[str]] = None,
+        args: List[str] | None = None,
+        end_of_options_args: List[str] | None = None,
         cwd: str = "",
         envs: Optional[Dict] = None,
     ):
         self.git_filepath = git_filepath
         self.command = command
         self.args = args or []
+        self.end_of_options_args = end_of_options_args or []
         self.cwd = cwd
         self.envs = envs or {}
 
     def to_cmd(self, obscure: bool = False) -> List[str]:
-        return [self.git_filepath, self.command, *self.args]
+        cmd = [self.git_filepath, self.command, *self.args]
+        if self.end_of_options_args:
+            cmd.append("--")
+            cmd.extend(self.end_of_options_args)
+        return cmd
 
     def get_sensitive_texts(self) -> List[str]:
         """Get sensitive texts in the current command, the texts must not be displayed
@@ -93,13 +112,14 @@ def __init__(
         SYNOPSIS
                git clone [--bare] [--] <repository> [<target_directory>]
         """
-        super().__init__(git_filepath, "clone", args, cwd, envs)
+        super().__init__(git_filepath=git_filepath, command="clone", args=args, cwd=cwd, envs=envs)
         self.repository = repository
         self.target_directory = target_directory
 
     def to_cmd(self, obscure: bool = False) -> List[str]:
         """"""
         cmd = super().to_cmd(obscure=obscure)
+        cmd.append("--")
         cmd.extend([str(self.repository.obscure()) if obscure else str(self.repository), self.target_directory])
         return cmd
 
@@ -128,22 +148,26 @@ class GitClient:
 
     def checkout(self, path: Path, target: str) -> str:
         """切换分支或tag"""
-        command = GitCommand(git_filepath=self._git_filepath, command="checkout", args=[target], cwd=str(path))
+        command = GitCommand(
+            git_filepath=self._git_filepath, command="checkout", end_of_options_args=[target], cwd=str(path)
+        )
         return self.run(command)
 
     def clone(
         self,
         url: Union[str, MutableURL],
         path: Path,
-        envs: Optional[dict] = None,
-        depth: Optional[int] = None,
-        branch: Optional[str] = None,
+        envs: Dict | None = None,
+        depth: int | None = None,
+        branch: str | None = None,
     ) -> str:
         """克隆仓库
 
         :param url: 仓库地址
         :param path: 存储路径
         :param envs: 环境变量
+        :param depth: 克隆深度
+        :param branch: 克隆分支
         :return: 返回 clone 结果
         """
         args = []
@@ -182,8 +206,7 @@ def clone_no_blob(self, url: Union[str, MutableURL], path: Path) -> str:
     def list_remote(self, url: Union[str, MutableURL]) -> List[Tuple[str, str]]:
         """通过 ls-remote 命令，直接获取远端仓库的 branch 与 tag 等信息。该命令不依赖本地文件系统。
 
-        :param path: 项目路径
-        :param remote: 远端名称，默认为 origin
+        :param url: 仓库地址
         :return: 命令执行结果，包含 (commit_id, ref) 的列表
         """
         results = []
@@ -206,12 +229,20 @@ def list_remote(self, url: Union[str, MutableURL]) -> List[Tuple[str, str]]:
     def list_remote_raw(self, url: Union[str, MutableURL]) -> str:
         """通过 ls-remote 命令，直接获取远端仓库的 branch 与 tag 等信息。返回命令原始执行结果。
 
-        :param path: 项目路径
-        :param remote: 远端名称，默认为 origin
+        :param url: 仓库地址
         :return: 命令原始执行结果
         """
+        # 在执行 Git 操作前校验 URL 安全性，防止历史脏数据绕过
+        url_str = str(url)
+        try:
+            validate_repo_url(url_str)
+        except ValueError as e:
+            raise GitURLValidationError(str(e))
+
         # The "cwd" is pointless for running this command, always use current dir.
-        command = GitCommand(git_filepath=self._git_filepath, command="ls-remote", args=[str(url)], cwd=os.getcwd())
+        command = GitCommand(
+            git_filepath=self._git_filepath, command="ls-remote", end_of_options_args=[url_str], cwd=os.getcwd()
+        )
         return self.run(command)
 
     def list_refs(self, path: Path) -> Generator[Ref, None, None]:

apiserver/paasng/paasng/platform/sourcectl/serializers.py

@@ -27,7 +27,7 @@
 from paasng.platform.sourcectl.type_specs import BkSvnSourceTypeSpec
 from paasng.platform.sourcectl.validators import validate_download_url
 from paasng.utils.serializers import DecryptableJSONField, SourceControlField, UserNameField, VerificationCodeField
-from paasng.utils.validators import validate_image_repo, validate_repo_url
+from paasng.utils.validators import SafeFilenameValidator, validate_image_repo, validate_repo_url
 
 logger = logging.getLogger(__name__)
 
@@ -206,5 +206,13 @@ def validate_package_url(self, value: str) -> str:
 
 
 class SourcePackageUploadViaFileSLZ(serializers.Serializer):
-    package = serializers.FileField(help_text="源码包文件")
+    package = serializers.FileField(
+        help_text="源码包文件",
+        validators=[
+            SafeFilenameValidator(
+                name_pattern=r"[a-zA-Z0-9\-_.]+",
+                name_pattern_message="格式错误，只能包含字母(a-zA-Z)、数字(0-9)和半角连接符(-)、下划线(_)和点(.)",
+            )
+        ],
+    )
     allow_overwrite = serializers.BooleanField(help_text="是否允许覆盖原有的源码包", default=False, allow_null=True)

apiserver/paasng/paasng/platform/sourcectl/svn/admin.py

@@ -62,12 +62,13 @@ def del_authz(self, repo_path, user_or_group, type_id):
         raise NotImplementedError
 
 
+# FIXME: 该服务数年前已下线，后续确认能否整个删除
 class BaseRealSvnAuthClient(BaseSvnAuthClient):
-    SVN_SECRET = "32fc6114554e3c53d5952594510021e2"
+    SVN_SECRET = ""
     SVN_OPERATE_ERROR_NOTIFIER = settings.ADMIN_USERNAME
     DUMMY = True
     TIMEOUT = 60
-    SSL_VERIFY = False
+    SSL_VERIFY = True
 
     BASE_SVN_ADD_USER = "{admin_url}svn_add/user/"
     BASE_SVN_MOD_COMMON = "{admin_url}svn_mod/common/"

apiserver/paasng/paasng/settings/__init__.py

@@ -129,8 +129,6 @@ def minimum_database_version(self):
 
 DEBUG = settings.get("DEBUG", False)
 
-SESSION_COOKIE_HTTPONLY = False
-
 RUNNING_TESTS = "test" in sys.argv or "pytest" in sys.argv[0] or "PYTEST_XDIST_TESTRUNUID" in os.environ
 
 INSTALLED_APPS = [